AkkomaGang/elasticsearch-elixir
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Rework configuration

Browse source 
Also, use more understandable examples.
This commit is contained in:
Daniel Berkompas 2017-11-03 15:17:58 -07:00
parent 832dc9db99
commit d30323d103
16 changed files with 126 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -64,7 +64,7 @@ config :elasticsearch,
# This file describes the mappings and settings for your index. It will
# be posted as-is to Elasticsearch when you create your index, and
# therefore allows all the settings you could post directly.
schema: "priv/elasticsearch/cities.json",
settings: "priv/elasticsearch/cities.json",
# This is the list of data sources that should be used to populate this
# index. The `:loader` module above will be passed each one of these

13
config/config.exs
View file

@ -10,16 +10,9 @@ config :elasticsearch,
bulk_wait_interval: 15_000, # 15 seconds
api_module: Elasticsearch.API.HTTP,
indexes: %{
index1: %{
alias: "index1_alias",
schema: "test/support/settings/index1.json",
posts: %{
settings: "test/support/settings/posts.json",
loader: Elasticsearch.Test.DataLoader,
sources: [Type1]
sources: [Post]
},
index2: %{
alias: "index2_alias",
schema: "test/support/settings/index2.json",
loader: Elasticsearch.Test.DataLoader,
sources: [Type2]
}
}

101
lib/elasticsearch.ex
View file

@ -14,10 +14,7 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index("test1", "{}")
:ok
iex> Elasticsearch.create_index("test1", %{})
iex> Elasticsearch.create_index("posts-1", "{}")
:ok
"""
@spec create_index(String.t, map | String.t) ::
@ -32,10 +29,10 @@ defmodule Elasticsearch do
## Example
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
:ok
iex> Elasticsearch.create_index_from_file("test2", "nonexistent.json")
iex> Elasticsearch.create_index_from_file("posts-1", "nonexistent.json")
{:error, :enoent}
"""
@spec create_index_from_file(String.t, Path.t) ::
@ -55,13 +52,13 @@ defmodule Elasticsearch do
## Example
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> struct = %Type1{id: 123, name: "Post", author: "Author"}
...> Elasticsearch.put_document(struct, "test1")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> struct = %Post{id: 123, title: "Post", author: "Author"}
...> Elasticsearch.put_document(struct, "posts-1")
{:ok,
%{"_id" => "123", "_index" => "test1",
%{"_id" => "123", "_index" => "posts-1",
"_shards" => %{"failed" => 0, "successful" => 1, "total" => 2},
"_type" => "type1", "_version" => 1, "created" => true,
"_type" => "post", "_version" => 1, "created" => true,
"result" => "created"}}
"""
@spec put_document(Document.t, String.t) :: response
@ -113,8 +110,8 @@ defmodule Elasticsearch do
## Example
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.alias_index("test1", "test")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.alias_index("posts-1", "posts")
:ok
"""
@spec alias_index(String.t, String.t) ::
@ -173,13 +170,17 @@ defmodule Elasticsearch do
## Example
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.create_index_from_file("test2", "test/support/settings/index2.json")
...> Elasticsearch.indexes_starting_with("test")
{:ok, ["test1", "test2"]}
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.indexes_starting_with("posts")
{:ok, ["posts-1"]}
"""
@spec indexes_starting_with(String.t | atom) ::
{:ok, [String.t]} |
{:error, Elasticsearch.Exception.t}
def indexes_starting_with(prefix) do
with {:ok, indexes} <- get("/_cat/indices?format=json") do
prefix = to_string(prefix)
indexes =
indexes
|> Enum.map(&(&1["index"]))
@ -195,17 +196,17 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.create_index_from_file("test2", "test/support/settings/index2.json")
...> Elasticsearch.latest_index_starting_with("test")
{:ok, "test2"}
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.create_index_from_file("posts-2", "test/support/settings/posts.json")
...> Elasticsearch.latest_index_starting_with("posts")
{:ok, "posts-2"}
If there are no indexes matching that prefix:
iex> Elasticsearch.latest_index_starting_with("nonexistent")
{:error, :not_found}
"""
@spec latest_index_starting_with(String.t) ::
@spec latest_index_starting_with(String.t | atom) ::
{:ok, String.t} |
{:error, :not_found} |
{:error, Elasticsearch.Exception.t}
@ -228,8 +229,8 @@ defmodule Elasticsearch do
## Example
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.refresh_index("test1")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.refresh_index("posts-1")
:ok
"""
@spec refresh_index(String.t) :: :ok | {:error, Elasticsearch.Exception.t}
@ -244,8 +245,8 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.refresh_index!("test1")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.refresh_index!("posts-1")
:ok
iex> Elasticsearch.refresh_index!("nonexistent")
@ -270,17 +271,17 @@ defmodule Elasticsearch do
If there is only one index, and `num_to_keep` is >= 1, the index is not deleted.
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.clean_indexes_starting_with("test", 1)
...> Elasticsearch.indexes_starting_with("test")
{:ok, ["test1"]}
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.clean_indexes_starting_with("posts", 1)
...> Elasticsearch.indexes_starting_with("posts")
{:ok, ["posts-1"]}
If `num_to_keep` is less than the number of indexes, the older indexes are
deleted.
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.clean_indexes_starting_with("test", 0)
...> Elasticsearch.indexes_starting_with("test")
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.clean_indexes_starting_with("posts", 0)
...> Elasticsearch.indexes_starting_with("posts")
{:ok, []}
"""
@spec clean_indexes_starting_with(String.t, integer) ::
@ -363,15 +364,15 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.put("/test1/type1/id", %{"name" => "name", "author" => "author"})
iex> Elasticsearch.create_index_from_file("posts-1", "test/support/settings/posts.json")
...> Elasticsearch.put("/posts-1/post/id", %{"title" => "title", "author" => "author"})
{:ok,
%{"_id" => "id", "_index" => "test1",
%{"_id" => "id", "_index" => "posts-1",
"_shards" => %{"failed" => 0, "successful" => 1, "total" => 2},
"_type" => "type1", "_version" => 1, "created" => true,
"_type" => "post", "_version" => 1, "created" => true,
"result" => "created"}}
iex> Elasticsearch.put("/bad/url", %{"name" => "name", "author" => "author"})
iex> Elasticsearch.put("/bad/url", %{"title" => "title", "author" => "author"})
{:error,
%Elasticsearch.Exception{col: nil, line: nil,
message: "No handler found for uri [/bad/url] and method [PUT]",
@ -388,11 +389,11 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.put!("/test1/type1/id", %{"name" => "name", "author" => "author"})
%{"_id" => "id", "_index" => "test1",
iex> Elasticsearch.create_index_from_file("posts", "test/support/settings/posts.json")
...> Elasticsearch.put!("/posts/post/id", %{"name" => "name", "author" => "author"})
%{"_id" => "id", "_index" => "posts",
"_shards" => %{"failed" => 0, "successful" => 1, "total" => 2},
"_type" => "type1", "_version" => 1, "created" => true,
"_type" => "post", "_version" => 1, "created" => true,
"result" => "created"}
iex> Elasticsearch.put!("/bad/url", %{"data" => "here"})
@ -411,9 +412,9 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
iex> Elasticsearch.create_index_from_file("posts", "test/support/settings/posts.json")
...> query = %{"query" => %{"match_all" => %{}}}
...> {:ok, resp} = Elasticsearch.post("/test1/_search", query)
...> {:ok, resp} = Elasticsearch.post("/posts/_search", query)
...> resp["hits"]["hits"]
[]
"""
@ -427,9 +428,9 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
iex> Elasticsearch.create_index_from_file("posts", "test/support/settings/posts.json")
...> query = %{"query" => %{"match_all" => %{}}}
...> resp = Elasticsearch.post!("/test1/_search", query)
...> resp = Elasticsearch.post!("/posts/_search", query)
...> is_map(resp)
true
@ -451,8 +452,8 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.delete("/test1")
iex> Elasticsearch.create_index_from_file("posts", "test/support/settings/posts.json")
...> Elasticsearch.delete("/posts")
{:ok, %{"acknowledged" => true}}
It returns an error if the given resource does not exist.
@ -483,8 +484,8 @@ defmodule Elasticsearch do
## Examples
iex> Elasticsearch.create_index_from_file("test1", "test/support/settings/index1.json")
...> Elasticsearch.delete!("/test1")
iex> Elasticsearch.create_index_from_file("posts", "test/support/settings/posts.json")
...> Elasticsearch.delete!("/posts")
%{"acknowledged" => true}
Raises an error if the resource is invalid.

8
lib/elasticsearch/builder.ex
View file

@ -16,12 +16,12 @@ defmodule Elasticsearch.Builder do
## Example
iex> file = "test/support/settings/index1.json"
iex> file = "test/support/settings/posts.json"
...> loader = Elasticsearch.Test.DataLoader
...> Builder.hot_swap_index("index1", file, loader, [Type1])
...> Builder.hot_swap_index("posts", file, loader, [Post])
:ok
"""
@spec hot_swap_index(String.t, Path.t, Elasticsearch.DataLoader.t, list) ::
@spec hot_swap_index(String.t | atom, String.t, Elasticsearch.DataLoader.t, list) ::
:ok |
{:error, Elasticsearch.Exception.t}
def hot_swap_index(alias, settings_file, loader, sources) do
@ -45,7 +45,7 @@ defmodule Elasticsearch.Builder do
Config.build_index_name("main")
# => "main-1509581256"
"""
@spec build_index_name(String.t) :: String.t
@spec build_index_name(String.t | atom) :: String.t
def build_index_name(alias) do
"#{alias}-#{system_timestamp()}"
end

19
lib/elasticsearch/bulk.ex
View file

@ -16,10 +16,10 @@ defmodule Elasticsearch.Bulk do
## Examples
iex> Bulk.encode(%Type1{id: "my-id"}, "my-index")
iex> Bulk.encode(%Post{id: "my-id"}, "my-index")
{:ok, \"\"\"
{"create":{"_type":"type1","_index":"my-index","_id":"my-id"}}
{"name":null,"author":null}
{"create":{"_type":"post","_index":"my-index","_id":"my-id"}}
{"title":null,"author":null}
\"\"\"}
iex> Bulk.encode(123, "my-index")
@ -42,14 +42,14 @@ defmodule Elasticsearch.Bulk do
## Example
iex> Bulk.encode!(%Type1{id: "my-id"}, "my-index")
iex> Bulk.encode!(%Post{id: "my-id"}, "my-index")
\"\"\"
{"create":{"_type":"type1","_index":"my-index","_id":"my-id"}}
{"name":null,"author":null}
{"create":{"_type":"post","_index":"my-index","_id":"my-id"}}
{"title":null,"author":null}
\"\"\"
iex> Bulk.encode!(123, "my-index")
** (Protocol.UndefinedError) protocol Elasticsearch.Document not implemented for 123. This protocol is implemented for: Type1, Type2
** (Protocol.UndefinedError) protocol Elasticsearch.Document not implemented for 123. This protocol is implemented for: Post
"""
def encode!(struct, index) do
header = header("create", index, struct)
@ -65,11 +65,6 @@ defmodule Elasticsearch.Bulk do
@doc """
Uploads all the data from the list of `sources` to the given index.
Data for each `source` will be fetched using the configured `:loader`.
## Example
iex> Bulk.upload("test1", Elasticsearch.Test.DataLoader, [Type1])
:ok
"""
@spec upload(String.t, Elasticsearch.DataLoader.t, list) :: :ok | {:error, [map]}
def upload(index_name, loader, sources, errors \\ [])

18
lib/elasticsearch/config.ex
View file

@ -66,26 +66,24 @@ defmodule Elasticsearch.Config do
config :elasticsearch,
indexes: %{
index1: %{
alias: "index1_alias",
schema: "test/support/settings/index1.json",
sources: [Type1]
posts: %{
settings: "test/support/settings/posts.json",
loader: Elasticsearch.Test.DataLoader,
sources: [Post]
}
}
## Example
iex> Config.config_for_index(:index1)
iex> Config.config_for_index(:posts)
%{
alias: "index1_alias",
schema: "test/support/settings/index1.json",
settings: "test/support/settings/posts.json",
loader: Elasticsearch.Test.DataLoader,
sources: [Type1]
sources: [Post]
}
"""
@spec config_for_index(atom) :: %{
alias: String.t,
schema: String.t,
settings: String.t,
loader: DataLoader.t,
sources: [DataLoader.source]
} | nil

18
lib/mix/elasticsearch.build.ex
View file

@ -16,25 +16,25 @@ defmodule Mix.Tasks.Elasticsearch.Build do
{indexes, type} = parse_args!(args)
for index <- indexes do
config = Config.config_for_index(index)
build(config, type)
for alias <- indexes do
config = Config.config_for_index(alias)
build(alias, config, type)
end
end
defp build(config, :existing) do
case Elasticsearch.latest_index_starting_with(config[:alias]) do
defp build(alias, config, :existing) do
case Elasticsearch.latest_index_starting_with(alias) do
{:ok, index_name} ->
IO.puts("Index already exists: #{index_name}")
{:error, :not_found} ->
build(config, :rebuild)
build(alias, config, :rebuild)
{:error, exception} ->
Mix.raise(exception)
end
end
defp build(%{alias: alias, schema: schema, loader: loader, sources: sources}, :rebuild) do
with :ok <- Builder.hot_swap_index(alias, schema, loader, sources) do
defp build(alias, %{settings: settings, loader: loader, sources: sources}, :rebuild) do
with :ok <- Builder.hot_swap_index(alias, settings, loader, sources) do
:ok
else
{:error, errors} when is_list(errors) ->
@ -48,7 +48,7 @@ defmodule Mix.Tasks.Elasticsearch.Build do
"""
{:error, :enoent} ->
Mix.raise """
Schema file not found at #{schema}.
Schema file not found at #{settings}.
"""
{:error, exception} ->
Mix.raise """

6
test/elasticsearch/builder_test.exs
View file

@ -4,4 +4,10 @@ defmodule Elasticsearch.BuilderTest do
alias Elasticsearch.Builder
doctest Elasticsearch.Builder
setup do
for index <- ["posts"] do
Elasticsearch.delete("/#{index}")
end
end
end

8
test/elasticsearch_test.exs
View file

@ -5,9 +5,11 @@ defmodule ElasticsearchTest do
setup do
on_exit fn ->
for index <- ["test1", "test2", "nonexistent"] do
Elasticsearch.delete("/#{index}")
end
"posts"
|> Elasticsearch.indexes_starting_with()
|> elem(1)
|> Enum.map(&Elasticsearch.delete!("/#{&1}"))
Elasticsearch.delete("/nonexistent")
end
end
end

23
test/mix/elasticsearch.build_test.exs
View file

@ -8,9 +8,10 @@ defmodule Mix.Tasks.Elasticsearch.BuildTest do
setup do
on_exit fn ->
for index <- ["index1_alias"] do
Elasticsearch.delete("/#{index}")
end
"posts"
|> Elasticsearch.indexes_starting_with()
|> elem(1)
|> Enum.map(&Elasticsearch.delete("/#{&1}"))
end
end
@ -34,35 +35,35 @@ defmodule Mix.Tasks.Elasticsearch.BuildTest do
end
test "builds configured index" do
rerun("elasticsearch.build", ["index1"])
rerun("elasticsearch.build", ["posts"])
resp = Elasticsearch.get!("/index1_alias/_search")
resp = Elasticsearch.get!("/posts/_search")
assert resp["hits"]["total"] == 10_000
end
test "only keeps two index versions" do
for _ <- 1..3 do
rerun("elasticsearch.build", ["index1"])
rerun("elasticsearch.build", ["posts"])
:timer.sleep(1000)
end
{:ok, indexes} = Elasticsearch.indexes_starting_with("index1")
{:ok, indexes} = Elasticsearch.indexes_starting_with("posts")
assert length(indexes) == 2
[_previous, current] = Enum.sort(indexes)
# assert that the most recent index is the one that is aliased
assert {:ok, %{^current => _}} = Elasticsearch.get("/index1_alias/_alias")
assert {:ok, %{^current => _}} = Elasticsearch.get("/posts/_alias")
end
test "--existing checks if index exists" do
rerun("elasticsearch.build", ["index1"])
rerun("elasticsearch.build", ["posts"])
io =
capture_io fn ->
rerun("elasticsearch.build", ["index1", "--existing"])
rerun("elasticsearch.build", ["posts", "--existing"])
end
assert io =~ "Index already exists: index1_alias-"
assert io =~ "Index already exists: posts-"
end
end
end

4
test/support/data_loader.ex
View file

@ -2,8 +2,8 @@ defmodule Elasticsearch.Test.DataLoader do
@moduledoc false
@behaviour Elasticsearch.DataLoader
def load(Type1, _offset, limit) when limit <= 10_000 do
[%Type1{name: "Name", author: "Author"}]
def load(Post, _offset, limit) when limit <= 10_000 do
[%Post{title: "Name", author: "Author"}]
|> Stream.cycle()
|> Stream.map(&Map.put(&1, :id, random_str()))
|> Enum.take(5000)

16
test/support/post.ex Normal file
View file

@ -0,0 +1,16 @@
defmodule Post do
@moduledoc false
defstruct id: nil, title: nil, author: nil
end
defimpl Elasticsearch.Document, for: Post do
def id(item), do: item.id
def type(_item), do: "post"
def parent(_item), do: false
def encode(item) do
%{
title: item.title,
author: item.author
}
end
end

14
test/support/settings/index2.json
View file

@ -1,14 +0,0 @@
{
"mappings": {
"type2": {
"properties": {
"name": {
"type": "string"
},
"author": {
"type": "string"
}
}
}
}
}

4
test/support/settings/index1.json → test/support/settings/posts.json
View file

@ -1,8 +1,8 @@
{
"mappings": {
"type1": {
"post": {
"properties": {
"name": {
"title": {
"type": "string"
},
"author": {

16
test/support/type1.ex
View file

@ -1,16 +0,0 @@
defmodule Type1 do
@moduledoc false
defstruct id: nil, name: nil, author: nil
end
defimpl Elasticsearch.Document, for: Type1 do
def id(item), do: item.id
def type(_item), do: "type1"
def parent(_item), do: false
def encode(item) do
%{
name: item.name,
author: item.author
}
end
end

16
test/support/type2.ex
View file

@ -1,16 +0,0 @@
defmodule Type2 do
@moduledoc false
defstruct name: nil, author: nil
end
defimpl Elasticsearch.Document, for: Type2 do
def id(item), do: item.name
def type(_item), do: "type1"
def parent(_item), do: false
def encode(item) do
%{
name: item.name,
author: item.author
}
end
end