From debcadf495f8488fe4dcc4884ea3c0b573832d7f Mon Sep 17 00:00:00 2001 From: Jordan Bracco Date: Tue, 16 Jun 2020 11:55:04 +0200 Subject: [PATCH] API cleanup, C improvements, docs, ... --- .gitignore | 4 +- .travis.yml | 6 +- CHANGELOG.md | 5 +- README.md | 70 ++++++++++++--------- lib/majic.ex | 49 ++++++++++----- lib/majic/config.ex | 2 + lib/majic/{helpers.ex => once.ex} | 14 ++--- lib/majic/pool.ex | 8 ++- lib/majic/server.ex | 61 +++++++++--------- mix.exs | 18 ++++-- mix.lock | 16 +++-- src/libmagic_port.c | 100 ++++++++++++++++++++---------- test/majic/helpers_test.exs | 13 ++-- test/majic/majic_test.exs | 6 ++ test/majic/pool_test.exs | 1 + test/majic/port_test.exs | 63 +++++++++++++++---- test/test_helper.exs | 26 +++++++- 17 files changed, 306 insertions(+), 156 deletions(-) rename lib/majic/{helpers.ex => once.ex} (57%) diff --git a/.gitignore b/.gitignore index 7b641ec..6356d56 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ erl_crash.dump *.ez *.beam /config/*.secret.exs -/priv/apprentice -/src/*.o +/priv/libmagic_port /test/*.mgc +core.* diff --git a/.travis.yml b/.travis.yml index 1c68e47..f3a9db8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,15 +4,15 @@ elixir: - 1.10 - 1.9 - 1.8 - - 1.7 otp_release: - - '22.2' + - '22.3' + - '23.0.2' before_script: - mix compile –warnings-as-errors - mix credo --strict - - if [[ "$TRAVIS_ELIXIR_VERSION" =~ "1.10" ]]; then mix format mix.exs "{config,clients,games,lib,test}/**/*.{ex,exs}" --check-formatted; fi + - if [[ "$TRAVIS_ELIXIR_VERSION" =~ "1.10" ]]; then mix format mix.exs "{config,lib,test}/**/*.{ex,exs}" --check-formatted; fi before_install: - sudo apt-get install -y build-essential erlang-dev libmagic-dev diff --git a/CHANGELOG.md b/CHANGELOG.md index 43affe4..c49c78a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,12 +18,13 @@ The format is based on [Keep a Changelog][1], and this project adheres to [Seman ## Changed - C port now using erl_interface +- Builds on Musl +- Better error and timeout handling - `Majic.Server.reload/2,3` - `Majic.Server.recycle/2,3` - Bytes support: `Majic.Server.perform(ref, {:bytes, <<>>})` -- Builds on Musl -- Better error and timeout handling - Renamed `priv/apprentice` to `priv/libmagic_port` to be more obvious in `ps` +- Renamed `Majic.Helpers.perform_once` to `Majic.Once.perform` ## gen_majic [1.0] diff --git a/README.md b/README.md index e38ebb8..28ed902 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Compilation of the underlying C program is automatic and handled by [elixir_make Depending on the use case, you may utilise a single (one-off) Majic process without reusing it as a daemon, or utilise a connection pool (such as Poolboy) in your application to run multiple persistent Majic processes. -To use Majic directly, you can use `Majic.Helpers.perform_once/1`: +To use Majic directly, you can use `Majic.Once.perform/1`: ```elixir iex(1)> Majic.perform(".", once: true) @@ -67,11 +67,23 @@ When using `Majic.Server.start_link/1` to start a persistent server, or `Majic.H See `t:Majic.Server.option/0` for details. +__Note__ `:recycle_thresold` is only useful if you are using a libmagic `<5.29`, where it was susceptible to memleaks +([details](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=840754)]). In future versions of `majic` this option could +be ignored. + +### Reloading / Altering databases + +If you want `majic` to reload its database(s), run `Majic.Server.reload(ref)`. + +If you want to add or remove databases to a running server, you would have to run `Majic.Server.reload(ref, databases)` +where databases being the same argument as `database_patterns` on start. `Majic` does not support adding/removing +databases at runtime without a port reload. + ### Use Cases -### Ad-Hoc Requests +#### Ad-Hoc Requests -For ad-hoc requests, you can use the helper method `Majic.Helpers.perform_once/2`: +For ad-hoc requests, you can use the helper method `Majic.Once.perform_once/2`: ```elixir iex(1)> Majic.perform(Path.join(File.cwd!(), "Makefile"), once: true) @@ -83,63 +95,64 @@ iex(1)> Majic.perform(Path.join(File.cwd!(), "Makefile"), once: true) }} ``` -### Supervised Requests +#### Supervised Requests The Server should be run under a supervisor which provides resiliency. -Here we run it under a supervisor: +Here we run it under a supervisor in an application: ```elixir -iex(1)> {:ok, pid} = Supervisor.start_link([{Majic.Server, name: :gen_magic}], strategy: :one_for_one) -{:ok, #PID<0.199.0>} +children = + [ + # ... + {Majic.Server, [name: YourApp.Majic]} + ] + +opts = [strategy: :one_for_one, name: YourApp.Supervisor] +Supervisor.start_link(children, opts) ``` Now we can ask it to inspect a file: ```elixir -iex(2)> Majic.perform(Path.expand("~/.bash_history"), server: :gen_magic) +iex(2)> Majic.perform(Path.expand("~/.bash_history"), server: YourApp.Majic) {:ok, %Majic.Result{mime_type: "text/plain", encoding: "us-ascii", content: "ASCII text"}} ``` Note that in this case we have opted to use a named process. -### Pool +#### Pool For concurrency *and* resiliency, you may start the `Majic.Pool`. By default, it will start a `Majic.Server` worker per online scheduler: You can add a pool in your application supervisor by adding it as a child: -``` - children = - [ - # ... - {Majic.Pool, [name: YourApp.MajicPool, pool_size: 2]} - ] +```elixir +children = + [ + # ... + {Majic.Pool, [name: YourApp.MajicPool, pool_size: 2]} + ] - opts = [strategy: :one_for_one, name: YourApp.Supervisor] - Supervisor.start_link(children, opts) +opts = [strategy: :one_for_one, name: YourApp.Supervisor] +Supervisor.start_link(children, opts) ``` And then you can use it with `Majic.perform/2` with `pool: YourApp.MajicPool` option: -``` +```elixir iex(1)> Majic.perform(Path.expand("~/.bash_history"), pool: YourApp.MajicPool) {:ok, %Majic.Result{mime_type: "text/plain", encoding: "us-ascii", content: "ASCII text"}} ``` -### Check Uploaded Files +#### Use with Plug.Upload -If you use Phoenix, you can inspect the file from your controller: +If you use Plug or Phoenix, you may want to automatically verify the content type of every `Plug.Upload`. The +`Majic.Plug` is there for this. -```elixir -def upload(conn, %{"upload" => %{path: path}}) do, - {:ok, result} = Majic.perform(path, server: :gen_magic) - text(conn, "Received your file containing #{result.content}") -end -``` - -Obviously, it will be more ideal if you have wrapped `Majic.Server` in a pool such as Poolboy, to avoid constantly starting and stopping the underlying C program. +Enable it by using `plug Majic.Plug, pool: YourApp.MajicPool` in your pipeline or controller. Then, every `Plug.Upload` +in `conn.params` is now verified. The filename is also altered with an extension matching its content-type. ## Notes @@ -165,3 +178,4 @@ thanks all contributors for their generosity: - Soak Testing - Matthias and Ced for helping the author with C oddities - [Hecate](https://github.com/Kleidukos) for laughing at aforementionned oddities +- majic for giving inspiration for the lib name (magic, majic, get it? hahaha..) diff --git a/lib/majic.ex b/lib/majic.ex index 0b1b2d0..6f8d530 100644 --- a/lib/majic.ex +++ b/lib/majic.ex @@ -1,34 +1,51 @@ defmodule Majic do + alias Majic.{Once, Pool, Result, Server} + + @moduledoc """ + Robust libmagic integration for Elixir. + """ + @doc """ Perform on `path`. An option of `server: ServerName`, `pool: PoolName` or `once: true` must be passed. """ - @type name :: {:pool, atom()} | {:server, GenMagic.Server.t()} | {:once, true} - @type option :: name + @type target :: Path.t() | {:bytes, binary()} + @type result :: {:ok, Result.t()} | {:error, term() | String.t()} + @type name :: {:pool, atom()} | {:server, Server.t()} | {:once, true} + @type option :: name | Server.start_option() | Pool.option() - @spec perform(GenMagic.Server.target(), [option()]) :: GenMagic.Server.result() - def perform(path, opts, timeout \\ 5000) do - mod = cond do - Keyword.has_key?(opts, :pool) -> {GenMagic.Pool, Keyword.get(opts, :pool)} - Keyword.has_key?(opts, :server) -> {GenMagic.Server, Keyword.get(opts, :server)} - Keyword.has_key?(opts, :once) -> {GenMagic.Helpers, nil} - true -> nil - end + @spec perform(target(), [option()]) :: result() + def perform(path, opts) do + mod = + cond do + Keyword.has_key?(opts, :pool) -> {Pool, Keyword.get(opts, :pool)} + Keyword.has_key?(opts, :server) -> {Server, Keyword.get(opts, :server)} + Keyword.has_key?(opts, :once) -> {Once, nil} + true -> nil + end + + opts = + opts + |> Keyword.drop([:pool, :server, :once]) if mod do - do_perform(mod, path, timeout) + do_perform(mod, path, opts) else {:error, :no_method} end end - defp do_perform({GenMagic.Helpers, _}, path, timeout) do - GenMagic.Helpers.perform_once(path, timeout) - end - - defp do_perform({mod, name}, path, timeout) do + defp do_perform({Server = mod, name}, path, opts) do + timeout = Keyword.get(opts, :timeout, Majic.Config.default_process_timeout()) mod.perform(name, path, timeout) end + defp do_perform({Once = mod, _}, path, opts) do + mod.perform(path, opts) + end + + defp do_perform({Pool = mod, name}, path, opts) do + mod.perform(name, path, opts) + end end diff --git a/lib/majic/config.ex b/lib/majic/config.ex index a591c37..dabdabd 100644 --- a/lib/majic/config.ex +++ b/lib/majic/config.ex @@ -6,6 +6,8 @@ defmodule Majic.Config do @process_timeout 30_000 @recycle_threshold :infinity + def default_process_timeout, do: @process_timeout + def get_port_name do {:spawn_executable, to_charlist(get_executable_name())} end diff --git a/lib/majic/helpers.ex b/lib/majic/once.ex similarity index 57% rename from lib/majic/helpers.ex rename to lib/majic/once.ex index f4eb3cd..9bf91c0 100644 --- a/lib/majic/helpers.ex +++ b/lib/majic/once.ex @@ -1,13 +1,13 @@ -defmodule Majic.Helpers do +defmodule Majic.Once do @moduledoc """ Contains convenience functions for one-off use. """ - alias Majic.Result alias Majic.Server - @spec perform_once(Path.t() | {:bytes, binary}, [Server.option()]) :: - {:ok, Result.t()} | {:error, term()} + @process_timeout Majic.Config.default_process_timeout() + + @spec perform(Majic.target(), [Server.start_option()], timeout()) :: Majic.result() @doc """ Runs a one-shot process without supervision. @@ -16,13 +16,13 @@ defmodule Majic.Helpers do ## Example - iex(1)> {:ok, result} = Majic.Helpers.perform_once(".") + iex(1)> {:ok, result} = Majic.Once.perform(".") iex(2)> result %Majic.Result{content: "directory", encoding: "binary", mime_type: "inode/directory"} """ - def perform_once(path, options \\ []) do + def perform(path, options \\ [], timeout \\ @process_timeout) do with {:ok, pid} <- Server.start_link(options), - {:ok, result} <- Server.perform(pid, path), + {:ok, result} <- Server.perform(pid, path, timeout), :ok <- Server.stop(pid) do {:ok, result} end diff --git a/lib/majic/pool.ex b/lib/majic/pool.ex index e28aca8..1804a86 100644 --- a/lib/majic/pool.ex +++ b/lib/majic/pool.ex @@ -2,6 +2,9 @@ defmodule Majic.Pool do @behaviour NimblePool @moduledoc "Pool of `Majic.Server`" + @type name :: atom() + @type option :: {:pool_timeout, timeout()} | {:timeout, timeout()} + def child_spec(opts) do %{ id: __MODULE__, @@ -17,9 +20,10 @@ defmodule Majic.Pool do NimblePool.start_link(worker: {__MODULE__, options}, pool_size: pool_size) end + @spec perform(name(), Majic.target(), [option()]) :: Majic.result() def perform(pool, path, opts \\ []) do - pool_timeout = Keyword.get(opts, :pool_timeout, 5000) - timeout = Keyword.get(opts, :timeout, 5000) + pool_timeout = Keyword.get(opts, :pool_timeout, Majic.Config.default_process_timeout()) + timeout = Keyword.get(opts, :timeout, Majic.Config.default_process_timeout()) NimblePool.checkout!( pool, diff --git a/lib/majic/server.ex b/lib/majic/server.ex index 2bc9a44..c2568ae 100644 --- a/lib/majic/server.ex +++ b/lib/majic/server.ex @@ -10,7 +10,8 @@ defmodule Majic.Server do alias Majic.Server.Data alias Majic.Server.Status import Kernel, except: [send: 2] - require Logger + @database_patterns [:default] + @process_timeout Majic.Config.default_process_timeout() @typedoc """ Represents the reference to the underlying server, as returned by `:gen_statem`. @@ -41,7 +42,7 @@ defmodule Majic.Server do Can be set to `:infinity` if you do not wish for the program to be recycled. - - `:database_patterns`: Specifies what magic databases to load; you can specify a list of either + - `:database_patterns`: Specifies what magic databases to load; you can specify a list of files, or of Path Patterns (see `Path.wildcard/2`) or `:default` to instruct the C program to load the appropriate databases. @@ -49,17 +50,13 @@ defmodule Majic.Server do [:default, "path/to/my/magic"] """ - @database_patterns [:default] - @type option :: + @type start_option :: {:name, atom() | :gen_statem.server_name()} | {:startup_timeout, timeout()} | {:process_timeout, timeout()} | {:recycle_threshold, non_neg_integer() | :infinity} | {:database_patterns, nonempty_list(:default | Path.t())} - @type target :: Path.t() | {:bytes, binary()} - @type result :: {:ok, Result.t()} | {:error, term() | String.t()} - @typedoc """ Current state of the Server: @@ -84,9 +81,9 @@ defmodule Majic.Server do """ @type state :: :starting | :processing | :available | :recycling - @spec child_spec([option()]) :: Supervisor.child_spec() - @spec start_link([option()]) :: :gen_statem.start_ret() - @spec perform(t(), target(), timeout()) :: result() + @spec child_spec([start_option()]) :: Supervisor.child_spec() + @spec start_link([start_option()]) :: :gen_statem.start_ret() + @spec perform(t(), Majic.target(), timeout()) :: Majic.result() @spec status(t(), timeout()) :: {:ok, Status.t()} | {:error, term()} @spec stop(t(), term(), timeout()) :: :ok @@ -125,7 +122,7 @@ defmodule Majic.Server do @doc """ Determines the type of the file provided. """ - def perform(server_ref, path, timeout \\ 5000) do + def perform(server_ref, path, timeout \\ @process_timeout) do case :gen_statem.call(server_ref, {:perform, path}, timeout) do {:ok, %Result{} = result} -> {:ok, result} {:error, reason} -> {:error, reason} @@ -135,21 +132,21 @@ defmodule Majic.Server do @doc """ Reloads a Server with a new set of databases. """ - def reload(server_ref, database_patterns \\ nil, timeout \\ 5000) do + def reload(server_ref, database_patterns \\ nil, timeout \\ @process_timeout) do :gen_statem.call(server_ref, {:reload, database_patterns}, timeout) end @doc """ Same as `reload/2,3` but with a full restart of the underlying C port. """ - def recycle(server_ref, database_patterns \\ nil, timeout \\ 5000) do + def recycle(server_ref, database_patterns \\ nil, timeout \\ @process_timeout) do :gen_statem.call(server_ref, {:recycle, database_patterns}, timeout) end @doc """ Returns status of the Server. """ - def status(server_ref, timeout \\ 5000) do + def status(server_ref, timeout \\ @process_timeout) do :gen_statem.call(server_ref, :status, timeout) end @@ -173,7 +170,7 @@ defmodule Majic.Server do data = %Data{ port_name: get_port_name(), - database_patterns: Keyword.get(options, :database_patterns, []), + database_patterns: Keyword.get(options, :database_patterns), port_options: get_port_options(options), startup_timeout: get_startup_timeout(options), process_timeout: get_process_timeout(options), @@ -223,6 +220,7 @@ defmodule Majic.Server do 1 -> :bad_db 2 -> :ei_error 3 -> :ei_bad_term + 4 -> :magic_error code -> {:unexpected_error, code} end @@ -237,14 +235,11 @@ defmodule Majic.Server do pattern -> Path.wildcard(pattern) end) - databases = - if databases == [] do - [:default] - else - databases - end - - {:keep_state, {databases, data}, {:state_timeout, 0, :load}} + if databases == [] do + {:stop, {:error, :no_databases_to_load}, data} + else + {:keep_state, {databases, data}, {:state_timeout, 0, :load}} + end end @doc false @@ -258,11 +253,11 @@ defmodule Majic.Server do end @doc false - def loading(:state_timeout, :load, {[database | databases], data} = state) do + def loading(:state_timeout, :load, {[database | _databases], data} = state) do command = case database do :default -> {:add_default_database, nil} - path -> {:add_database, database} + path -> {:add_database, path} end send(data.port, command) @@ -274,16 +269,14 @@ defmodule Majic.Server do case :erlang.binary_to_term(response) do {:ok, :loaded} -> {:keep_state, {databases, data}, {:state_timeout, 0, :load}} + + {:error, :not_loaded} -> + {:stop, {:error, {:database_load_failed, database}}, data} end end @doc false - def loading(:info, {port, {:exit_status, 1}}, {[database | _], %{port: port} = data}) do - {:stop, {:error, {:database_not_found, database}}, data} - end - - @doc false - def loading({:call, from}, :status, {[database | _], data}) do + def loading({:call, from}, :status, {_, data}) do handle_status_call(from, :loading, data) end @@ -310,6 +303,8 @@ defmodule Majic.Server do arg = case path do path when is_binary(path) -> {:file, path} + # Truncate to 50 bytes + {:bytes, <>} -> {:bytes, bytes} {:bytes, bytes} -> {:bytes, bytes} end @@ -360,7 +355,7 @@ defmodule Majic.Server do end @doc false - def processing(:state_timeout, _, %{port: port, request: {_, from, _}} = data) do + def processing(:state_timeout, _, %{request: {_, from, _}} = data) do response = {:reply, from, {:error, :timeout}} {:next_state, :recycling, %{data | request: nil}, [response, :hibernate]} end @@ -397,7 +392,7 @@ defmodule Majic.Server do @doc false def recycling(:state_timeout, :close, data) do - {:stop, {:error, :port_close_failed}} + {:stop, {:error, :port_close_failed}, data} end @doc false diff --git a/mix.exs b/mix.exs index 4e9b699..2e0a321 100644 --- a/mix.exs +++ b/mix.exs @@ -11,6 +11,7 @@ defmodule Majic.MixProject do version: "1.0.0", elixir: "~> 1.7", elixirc_paths: elixirc_paths(Mix.env()), + elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())], start_permanent: Mix.env() == :prod, compilers: [:elixir_make] ++ Mix.compilers(), package: package(), @@ -32,7 +33,7 @@ defmodule Majic.MixProject do defp dialyzer do [ - plt_add_apps: [:mix, :iex, :ex_unit], + plt_add_apps: [:mix, :iex, :ex_unit, :plug, :mime], flags: ~w(error_handling no_opaque race_conditions underspecs unmatched_returns)a, ignore_warnings: "dialyzer-ignore-warnings.exs", list_unused_filters: true @@ -41,11 +42,13 @@ defmodule Majic.MixProject do defp deps do [ - {:credo, "~> 1.4.0", only: [:dev, :test], runtime: false}, + {:nimble_pool, "~> 0.1"}, + {:plug, "~> 1.0", optional: true}, + {:mime, "~> 1.0", optional: true}, + {:credo, "~> 1.4", only: [:dev, :test], runtime: false}, {:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, - {:elixir_make, "~> 0.4", runtime: false}, - {:nimble_pool, "~> 0.1"} + {:elixir_make, "~> 0.4", runtime: false} ] end @@ -53,8 +56,8 @@ defmodule Majic.MixProject do [ files: ~w(lib/gen_magic/* src/*.c Makefile), licenses: ["Apache 2.0"], - links: %{"GitHub" => "https://github.com/evadne/packmatic"}, - source_url: "https://github.com/evadne/packmatic" + links: %{"GitHub" => "https://github.com/hrefhref/majic"}, + source_url: "https://github.com/hrefhref/majic" ] end @@ -64,4 +67,7 @@ defmodule Majic.MixProject do extras: ["README.md", "CHANGELOG.md"] ] end + + defp warnings_as_errors(:dev), do: false + defp warnings_as_errors(_), do: true end diff --git a/mix.lock b/mix.lock index 3317183..d369655 100644 --- a/mix.lock +++ b/mix.lock @@ -2,15 +2,19 @@ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, "credo": {:hex, :credo, "1.4.0", "92339d4cbadd1e88b5ee43d427b639b68a11071b6f73854e33638e30a0ea11f5", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1fd3b70dce216574ce3c18bdf510b57e7c4c85c2ec9cad4bff854abaf7e58658"}, "dialyxir": {:hex, :dialyxir, "1.0.0", "6a1fa629f7881a9f5aaf3a78f094b2a51a0357c843871b8bc98824e7342d00a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "aeb06588145fac14ca08d8061a142d52753dbc2cf7f0d00fc1013f53f8654654"}, - "earmark": {:hex, :earmark, "1.4.4", "4821b8d05cda507189d51f2caeef370cf1e18ca5d7dfb7d31e9cafe6688106a4", [:mix], [], "hexpm", "1f93aba7340574847c0f609da787f0d79efcab51b044bb6e242cae5aca9d264d"}, + "earmark": {:hex, :earmark, "1.4.5", "62ffd3bd7722fb7a7b1ecd2419ea0b458c356e7168c1f5d65caf09b4fbdd13c8", [:mix], [], "hexpm", "b7d0e6263d83dc27141a523467799a685965bf8b13b6743413f19a7079843f4f"}, "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "erlexec": {:hex, :erlexec, "1.10.0", "cba7924cf526097d2082ceb0ec34e7db6bca2624b8f3867fb3fa89c4cf25d227", [:rebar3], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.21.3", "857ec876b35a587c5d9148a2512e952e24c24345552259464b98bfbb883c7b42", [:mix], [{:earmark, "~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0db1ee8d1547ab4877c5b5dffc6604ef9454e189928d5ba8967d4a58a801f161"}, + "ex_doc": {:hex, :ex_doc, "0.22.1", "9bb6d51508778193a4ea90fa16eac47f8b67934f33f8271d5e1edec2dc0eee4c", [:mix], [{:earmark, "~> 1.4.0", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "d957de1b75cb9f78d3ee17820733dc4460114d8b1e11f7ee4fd6546e69b1db60"}, "exexec": {:hex, :exexec, "0.2.0", "a6ffc48cba3ac9420891b847e4dc7120692fb8c08c9e82220ebddc0bb8d96103", [:mix], [{:erlexec, "~> 1.10", [hex: :erlexec, repo: "hexpm", optional: false]}], "hexpm"}, - "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, - "makeup": {:hex, :makeup, "1.0.1", "82f332e461dc6c79dbd82fbe2a9c10d48ed07146f0a478286e590c83c52010b5", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "49736fe5b66a08d8575bf5321d716bac5da20c8e6b97714fec2bcd6febcfa1f8"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, + "jason": {:hex, :jason, "1.2.1", "12b22825e22f468c02eb3e4b9985f3d0cb8dc40b9bd704730efa11abd2708c44", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b659b8571deedf60f79c5a608e15414085fa141344e2716fbd6988a084b5f993"}, + "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, + "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"}, + "plug": {:hex, :plug, "1.10.3", "c9cebe917637d8db0e759039cc106adca069874e1a9034fd6e3fdd427fd3c283", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "01f9037a2a1de1d633b5a881101e6a444bcabb1d386ca1e00bb273a1f1d9d939"}, + "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, } diff --git a/src/libmagic_port.c b/src/libmagic_port.c index da8afaf..9a289cd 100644 --- a/src/libmagic_port.c +++ b/src/libmagic_port.c @@ -1,5 +1,5 @@ // -// The Sorcerer’s Apprentice +// libmagic_port: The Sorcerer’s Apprentice // // To use this program, compile it with dynamically linked libmagic, as mirrored // at https://github.com/file/file. You may install it with apt-get, @@ -24,12 +24,15 @@ // Commands are sent to the program STDIN as an erlang term of `{Operation, // Argument}`, and response of `{:ok | :error, Response}`. // +// The program may exit with the following exit codes: +// - 1 if libmagic handles could not be opened, +// - 2 if something went wrong with ei_*, +// - 3 if you sent invalid term format, +// - 255 if the loop exited unexpectedly. +// // Invalid packets will cause the program to exit (exit code 3). This will // happen if your Erlang Term format doesn't match the version the program has -// been compiled with, or if you send a command too huge. -// -// The program may exit with exit code 3 if something went wrong with ei_* -// functions. +// been compiled with. // // Commands: // {:reload, _} :: :ready @@ -47,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +59,7 @@ #include #define ERROR_OK 0 -#define ERROR_DB 1 +#define ERROR_MAGIC 1 #define ERROR_EI 2 #define ERROR_BAD_TERM 3 @@ -78,6 +82,7 @@ magic_t magic_setup(int flags); typedef char byte; void setup_environment(); +void magic_close_all(); void magic_open_all(); int magic_load_all(char *path); int process_command(uint16_t len, byte *buf); @@ -93,6 +98,8 @@ static magic_t magic_mime_type; // MAGIC_MIME_TYPE static magic_t magic_mime_encoding; // MAGIC_MIME_ENCODING static magic_t magic_type_name; // MAGIC_NONE +bool magic_loaded = false; + int main(int argc, char **argv) { EI_ENSURE(ei_init()); setup_environment(); @@ -143,37 +150,47 @@ int process_command(uint16_t len, byte *buf) { // {:file, path} if (strlen(atom) == 4 && strncmp(atom, "file", 4) == 0) { - char path[4097]; - ei_get_type(buf, &index, &termtype, &termsize); + if (magic_loaded) { + char path[4097]; + ei_get_type(buf, &index, &termtype, &termsize); - if (termtype == ERL_BINARY_EXT) { - if (termsize < 4096) { - long bin_length; - EI_ENSURE(ei_decode_binary(buf, &index, path, &bin_length)); - path[termsize] = '\0'; - process_file(path, &result); + if (termtype == ERL_BINARY_EXT) { + if (termsize < 4096) { + long bin_length; + EI_ENSURE(ei_decode_binary(buf, &index, path, &bin_length)); + path[termsize] = '\0'; + process_file(path, &result); + } else { + error(&result, "enametoolong"); + return 1; + } } else { - error(&result, "enametoolong"); + error(&result, "badarg"); return 1; } } else { - error(&result, "badarg"); + error(&result, "magic_database_not_loaded"); return 1; } // {:bytes, bytes} } else if (strlen(atom) == 5 && strncmp(atom, "bytes", 5) == 0) { - int termtype; - int termsize; - char bytes[51]; - EI_ENSURE(ei_get_type(buf, &index, &termtype, &termsize)); + if (magic_loaded) { + int termtype; + int termsize; + char bytes[51]; + EI_ENSURE(ei_get_type(buf, &index, &termtype, &termsize)); - if (termtype == ERL_BINARY_EXT && termsize < 50) { - long bin_length; - EI_ENSURE(ei_decode_binary(buf, &index, bytes, &bin_length)); - bytes[termsize] = '\0'; - process_bytes(bytes, termsize, &result); + if (termtype == ERL_BINARY_EXT && termsize < 50) { + long bin_length; + EI_ENSURE(ei_decode_binary(buf, &index, bytes, &bin_length)); + bytes[termsize] = '\0'; + process_bytes(bytes, termsize, &result); + } else { + error(&result, "badarg"); + return 1; + } } else { - error(&result, "badarg"); + error(&result, "magic_database_not_loaded"); return 1; } // {:add_database, path} @@ -190,7 +207,8 @@ int process_command(uint16_t len, byte *buf) { EI_ENSURE(ei_x_encode_atom(&result, "ok")); EI_ENSURE(ei_x_encode_atom(&result, "loaded")); } else { - exit(ERROR_DB); + EI_ENSURE(ei_x_encode_atom(&result, "error")); + EI_ENSURE(ei_x_encode_atom(&result, "not_loaded")); } } else { error(&result, "enametoolong"); @@ -207,7 +225,8 @@ int process_command(uint16_t len, byte *buf) { EI_ENSURE(ei_x_encode_atom(&result, "ok")); EI_ENSURE(ei_x_encode_atom(&result, "loaded")); } else { - exit(ERROR_DB); + EI_ENSURE(ei_x_encode_atom(&result, "error")); + EI_ENSURE(ei_x_encode_atom(&result, "not_loaded")); } // {:reload, _} } else if (strlen(atom) == 6 && strncmp(atom, "reload", 6) == 0) { @@ -230,25 +249,37 @@ int process_command(uint16_t len, byte *buf) { void setup_environment() { opterr = 0; } -void magic_open_all() { +void magic_close_all() { + magic_loaded = false; if (magic_mime_encoding) { magic_close(magic_mime_encoding); + magic_mime_encoding = NULL; } if (magic_mime_type) { magic_close(magic_mime_type); + magic_mime_type = NULL; } if (magic_type_name) { magic_close(magic_type_name); + magic_type_name = NULL; } +} + +void magic_open_all() { + magic_close_all(); magic_mime_encoding = magic_open(MAGIC_FLAGS_COMMON | MAGIC_MIME_ENCODING); magic_mime_type = magic_open(MAGIC_FLAGS_COMMON | MAGIC_MIME_TYPE); magic_type_name = magic_open(MAGIC_FLAGS_COMMON | MAGIC_NONE); - ei_x_buff ok_buf; - EI_ENSURE(ei_x_new_with_version(&ok_buf)); - EI_ENSURE(ei_x_encode_atom(&ok_buf, "ready")); - write_cmd(ok_buf.buff, ok_buf.index); - EI_ENSURE(ei_x_free(&ok_buf)); + if (magic_mime_encoding && magic_mime_type && magic_type_name) { + ei_x_buff ok_buf; + EI_ENSURE(ei_x_new_with_version(&ok_buf)); + EI_ENSURE(ei_x_encode_atom(&ok_buf, "ready")); + write_cmd(ok_buf.buff, ok_buf.index); + EI_ENSURE(ei_x_free(&ok_buf)); + } else { + exit(ERROR_MAGIC); + } } int magic_load_all(char *path) { @@ -263,6 +294,7 @@ int magic_load_all(char *path) { if ((res = magic_load(magic_type_name, path)) != 0) { return res; } + magic_loaded = true; return 0; } diff --git a/test/majic/helpers_test.exs b/test/majic/helpers_test.exs index ce514c5..ebe1736 100644 --- a/test/majic/helpers_test.exs +++ b/test/majic/helpers_test.exs @@ -1,9 +1,14 @@ -defmodule Majic.HelpersTest do +defmodule Majic.OnceTest do use Majic.MagicCase - doctest Majic.Helpers + doctest Majic.Once - test "perform_once" do + test "perform" do path = absolute_path("Makefile") - assert {:ok, %{mime_type: "text/x-makefile"}} = Majic.Helpers.perform_once(path) + assert {:ok, %{mime_type: "text/x-makefile"}} = Majic.Once.perform(path) + end + + test "Majic.perform" do + path = absolute_path("Makefile") + assert {:ok, %{mime_type: "text/x-makefile"}} = Majic.perform(path, once: true) end end diff --git a/test/majic/majic_test.exs b/test/majic/majic_test.exs index a47812d..b512bb3 100644 --- a/test/majic/majic_test.exs +++ b/test/majic/majic_test.exs @@ -11,6 +11,12 @@ defmodule MajicTest do assert {:ok, %{mime_type: "text/x-makefile"}} = Majic.Server.perform(pid, path) end + test "With Majic.perform" do + {:ok, pid} = Majic.Server.start_link([]) + path = absolute_path("Makefile") + assert {:ok, %{mime_type: "text/x-makefile"}} = Majic.perform(path, server: pid) + end + @tag external: true test "Load test local files" do {:ok, pid} = Majic.Server.start_link([]) diff --git a/test/majic/pool_test.exs b/test/majic/pool_test.exs index 12f8edf..38155e8 100644 --- a/test/majic/pool_test.exs +++ b/test/majic/pool_test.exs @@ -12,5 +12,6 @@ defmodule Majic.PoollTest do assert {:ok, _} = Majic.Pool.perform(TestPool, absolute_path("Makefile")) assert {:ok, _} = Majic.Pool.perform(TestPool, absolute_path("Makefile")) assert {:ok, _} = Majic.Pool.perform(TestPool, absolute_path("Makefile")) + assert {:ok, _} = Majic.perform(absolute_path("Makefile"), pool: TestPool) end end diff --git a/test/majic/port_test.exs b/test/majic/port_test.exs index 7c5f089..931d913 100644 --- a/test/majic/port_test.exs +++ b/test/majic/port_test.exs @@ -1,4 +1,4 @@ -defmodule Majic.ApprenticeTest do +defmodule Majic.PortTest do use Majic.MagicCase @tmp_path "/tmp/testgenmagicx" @@ -7,18 +7,10 @@ defmodule Majic.ApprenticeTest do test "sends ready" do port = Port.open(Majic.Config.get_port_name(), Majic.Config.get_port_options([])) on_exit(fn -> send(port, {self(), :close}) end) - assert_ready_and_init_default(port) + assert_ready(port) end - test "stops" do - port = Port.open(Majic.Config.get_port_name(), Majic.Config.get_port_options([])) - on_exit(fn -> send(port, {self(), :close}) end) - assert_ready_and_init_default(port) - send(port, {self(), {:command, :erlang.term_to_binary({:stop, :stop})}}) - assert_receive {^port, {:exit_status, 0}} - end - - test "exits with non existent database with an error" do + test "errors with non existent database with an error" do opts = [:use_stdio, :binary, :exit_status, {:packet, 2}, {:args, []}] port = Port.open(Majic.Config.get_port_name(), opts) on_exit(fn -> send(port, {self(), :close}) end) @@ -29,7 +21,54 @@ defmodule Majic.ApprenticeTest do {self(), {:command, :erlang.term_to_binary({:add_database, "/somewhere/nowhere"})}} ) - assert_receive {^port, {:exit_status, 1}} + assert_receive {^port, {:data, data}} + assert {:error, :not_loaded} == :erlang.binary_to_term(data) + end + + test "loads default database" do + opts = [:use_stdio, :binary, :exit_status, {:packet, 2}, {:args, []}] + port = Port.open(Majic.Config.get_port_name(), opts) + on_exit(fn -> send(port, {self(), :close}) end) + assert_ready(port) + + send( + port, + {self(), {:command, :erlang.term_to_binary({:add_default_database, nil})}} + ) + + assert_receive {^port, {:data, data}} + assert {:ok, :loaded} == :erlang.binary_to_term(data) + end + + test "reloads" do + opts = [:use_stdio, :binary, :exit_status, {:packet, 2}, {:args, []}] + port = Port.open(Majic.Config.get_port_name(), opts) + on_exit(fn -> send(port, {self(), :close}) end) + assert_ready_and_init_default(port) + + send(port, {self(), {:command, :erlang.term_to_binary({:reload, :reload})}}) + + assert_ready(port) + end + + test "errors when no database loaded" do + opts = [:use_stdio, :binary, :exit_status, {:packet, 2}, {:args, []}] + port = Port.open(Majic.Config.get_port_name(), opts) + on_exit(fn -> send(port, {self(), :close}) end) + assert_ready(port) + + send(port, {self(), {:command, :erlang.term_to_binary({:bytes, "hello world"})}}) + assert_receive {^port, {:data, data}} + assert {:error, :magic_database_not_loaded} = :erlang.binary_to_term(data) + refute_receive _ + end + + test "stops" do + port = Port.open(Majic.Config.get_port_name(), Majic.Config.get_port_options([])) + on_exit(fn -> send(port, {self(), :close}) end) + assert_ready_and_init_default(port) + send(port, {self(), {:command, :erlang.term_to_binary({:stop, :stop})}}) + assert_receive {^port, {:exit_status, 0}} end describe "port" do diff --git a/test/test_helper.exs b/test/test_helper.exs index 0b54ebe..8768806 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,7 +1,31 @@ ExUnit.start() +restore_ulimit = + case System.cmd("env", ["sh", "-c", "ulimit -c"]) do + {"unlimited\n", 0} -> + nil + + {old, 0} -> + case System.cmd("env", ["sh", "-c", "ulimit -c unlimited"]) do + {_, 0} -> + IO.puts("Enabled coredumps with ulimit.") + old + + error -> + IO.puts("Failed to enable coredumps: #{inspect(error)}") + end + + error -> + IO.puts("Couldn't use ulimit for coredumps: #{inspect(error)}") + nil + end + if System.get_env("TEAMCITY_VERSION") do ExUnit.configure(formatters: [TeamCityFormatter]) end -ExUnit.configure(exclude: [external: true]) +ExUnit.configure(exclude: [external: true], capture_log: true) + +if restore_ulimit do + System.cmd("env", ["sh", "-c", "ulimit -c #{String.trim(restore_ulimit)}"]) +end