Browse Source

feat(remote-parser): Logic in placec and used in Micropub.

jackyalcine 7 months ago
parent
commit
6cdbbca6ee
Signed by: Jacky Alciné <yo@jacky.wtf> GPG Key ID: 537A4F904B15268D
48 changed files with 1118 additions and 761 deletions
  1. 2
    1
      config/config.exs
  2. 1
    1
      config/test.exs
  3. 4
    7
      docker-compose.yml
  4. 20
    125
      lib/http.ex
  5. 4
    0
      lib/indieweb/mf2/remote.ex
  6. 5
    2
      lib/indieweb/micropub/entry.ex
  7. 31
    13
      lib/page.ex
  8. 0
    5
      lib/page/cacher.ex
  9. 0
    6
      lib/page/fetcher.ex
  10. 64
    0
      lib/page/parser.ex
  11. 81
    0
      lib/page/parser/activitystreams2.ex
  12. 20
    0
      lib/page/parser/json_linked_data.ex
  13. 39
    0
      lib/page/parser/microformats2.ex
  14. 23
    0
      lib/page/parser/open_graph.ex
  15. 5
    7
      lib/repo/webmention.ex
  16. 2
    0
      lib/storage.ex
  17. 3
    2
      lib/webmention.ex
  18. 9
    5
      mix.exs
  19. 11
    8
      mix.lock
  20. 364
    478
      package-lock.json
  21. 2
    2
      package.json
  22. 57
    0
      test/fixtures/vcr_cassettes/page_fetch_indieweb.json
  23. 63
    0
      test/fixtures/vcr_cassettes/page_fetch_success.json
  24. 45
    0
      test/fixtures/vcr_cassettes/page_fetch_unreachable.json
  25. 82
    0
      test/fixtures/vcr_cassettes/parse_activtystreams_parse_success.json
  26. 1
    1
      test/integration/controllers/contact_controller_test.exs
  27. 7
    0
      test/support/http.ex
  28. 1
    1
      test/support/steps/form.ex
  29. 1
    1
      test/unit/contact_test.exs
  30. 1
    76
      test/unit/http_test.exs
  31. 1
    1
      test/unit/indieweb/mf2/remote_test.exs
  32. 1
    1
      test/unit/indieweb/micropub/content_test.exs
  33. 1
    1
      test/unit/indieweb/micropub/entry_test.exs
  34. 1
    1
      test/unit/indieweb/micropub_test.exs
  35. 1
    1
      test/unit/indieweb/relme_test.exs
  36. 1
    1
      test/unit/indieweb/syndication_test.exs
  37. 60
    0
      test/unit/page/parser/activitystreams2_test.exs
  38. 0
    0
      test/unit/page/parser/json_linked_data_parser_test.exs
  39. 48
    0
      test/unit/page/parser/microformats2_test.exs
  40. 0
    0
      test/unit/page/parser/schema_org_parser_test.exs
  41. 1
    0
      test/unit/page/parser/twitter_card_parser_test.exs
  42. 0
    0
      test/unit/page/parser_test.exs
  43. 45
    1
      test/unit/page_test.exs
  44. 1
    1
      test/unit/repo/entry_test.exs
  45. 1
    1
      test/unit/repo/webmention_test.exs
  46. 5
    8
      test/unit/storage/json_test.exs
  47. 2
    2
      test/unit/webmention_test.exs
  48. 1
    1
      web/controllers/media_controller.ex

+ 2
- 1
config/config.exs View File

@@ -88,7 +88,8 @@ config :mnesia, dir: 'priv/repo/mnesia/#{Mix.env()}/#{node()}'
88 88
 
89 89
 config :indieweb,
90 90
   webmention_url_adapter: Koype.Webmention.URIAdapter,
91
-  http_adapter: Koype.Http.Adapter,
92 91
   auth_adapter: IndieWeb.Auth.Adapters.Default
93 92
 
93
+config :tesla, :adapter, Tesla.Adapter.Hackney
94
+
94 95
 import_config "#{Mix.env()}.exs"

+ 1
- 1
config/test.exs View File

@@ -1,6 +1,6 @@
1 1
 use Mix.Config
2 2
 
3
-config :logger, level: :info
3
+config :logger, level: :debug
4 4
 
5 5
 config :koype, Koype.Repo, pool: Ecto.Adapters.SQL.Sandbox
6 6
 

+ 4
- 7
docker-compose.yml View File

@@ -34,7 +34,7 @@ services:
34 34
     volumes:
35 35
       - ./:/opt/koype/:ro
36 36
   objectstorage:
37
-    image: "minio/minio:RELEASE.2019-03-13T21-59-47Z"
37
+    image: "minio/minio:RELEASE.2019-03-27T22-35-21Z"
38 38
     command: server /data
39 39
     restart: always
40 40
     volumes:
@@ -44,6 +44,7 @@ services:
44 44
       MINIO_ACCESS_KEY: ${OBJECT_STORAGE_ACCESS_KEY}
45 45
       MINIO_SECRET_KEY: ${OBJECT_STORAGE_SECRET_KEY}
46 46
       MINIO_REGION: "local"
47
+      MINIO_HTTP_TRACE: "-"
47 48
     networks:
48 49
       - network
49 50
   mc:
@@ -90,8 +91,8 @@ services:
90 91
     volumes:
91 92
       - ./:/opt/koype/:z
92 93
       - ./priv/repo/db/:/opt/koype/priv/repo/db/:z
93
-      - mix_deps:/opt/koype/deps/:rw
94
-      - mix_build:/opt/koype/_build/:rw
94
+      - ./deps:/opt/koype/deps/:rw
95
+      - ./_build:/opt/koype/_build/:rw
95 96
     healthcheck:
96 97
       test: ["CMD", "/tmp/koype-docker/healthcheck.sh"]
97 98
       interval: 60s
@@ -102,7 +103,3 @@ services:
102 103
 
103 104
 networks:
104 105
   network:
105
-
106
-volumes:
107
-  mix_deps:
108
-  mix_build:

+ 20
- 125
lib/http.ex View File

@@ -33,45 +33,18 @@ defmodule Koype.Http do
33 33
 
34 34
   defmodule Response do
35 35
     @moduledoc "Represents a HTTP response."
36
-    @enforce_keys [:code]
37
-    defstruct ~w(code body headers raw)a
36
+    @enforce_keys ~w(url raw)a
37
+    defstruct ~w(url code body headers raw)a
38 38
     @type t :: %Response{code: Integer.t(), body: Map.t(), headers: Map.t(), raw: any()}
39 39
   end
40 40
 
41 41
   defmodule Error do
42 42
     @moduledoc "Represents a HTTP error."
43
-    @enforce_keys [:reason]
43
+    @enforce_keys ~w(reason raw)a
44 44
     defstruct ~w(reason raw url)a
45 45
     @type t :: %Error{reason: any(), raw: any(), url: binary()}
46 46
   end
47 47
 
48
-  defmodule Adapter do
49
-    @moduledoc false
50
-    @behaviour IndieWeb.Http.Adapter
51
-
52
-    @impl true
53
-    def request(uri, method, opts) do
54
-      request_opts =
55
-        [
56
-          timeout: Keyword.get(opts, :timeout, Koype.Http.timeout()),
57
-          follow_redirects: true,
58
-          auto_sni: true,
59
-          headers: Keyword.get(opts, :headers, %{}) |> Map.to_list() || nil,
60
-          body: Keyword.get(opts, :body, %{}) |> URI.encode_query() || nil,
61
-          query: Keyword.get(opts, :query, nil)
62
-        ]
63
-        |> Enum.reject(fn {_, v} -> is_nil(v) end)
64
-
65
-      case Koype.Http.request(method, uri, request_opts) do
66
-        {:ok, resp} ->
67
-          {:ok, %IndieWeb.Http.Response{code: resp.code, headers: resp.headers, body: resp.body, raw: resp}}
68
-
69
-        {:error, resp} ->
70
-          {:error, %IndieWeb.Http.Error{message: resp.reason, raw: resp}}
71
-      end
72
-    end
73
-  end
74
-
75 48
   for method <- ~w(get post put patch delete head options)a do
76 49
     @doc "Helper method to dispatch a #{method} request."
77 50
     @spec unquote(method)(url :: String.t(), args :: Keyword.t()) :: {:ok, Response.t()} | {:error, Error.t()}
@@ -85,101 +58,23 @@ defmodule Koype.Http do
85 58
   def request(method, url, args \\ [])
86 59
   def request(method, %URI{} = uri, args), do: request(method, URI.to_string(uri), args)
87 60
 
88
-  def request(method, url, args) when is_binary(url) do
89
-    [
90
-      &do_request_potion/1,
91
-      &do_request_poison/1
92
-    ]
93
-    |> Enum.reduce_while({:error, :unspecified_network_error}, fn handler, acc ->
94
-      Logger.debug("Sending request to #{url} via method #{method}")
95
-      resp = handler.([url: url, method: method] ++ args)
96
-
97
-      case resp do
98
-        {:ok, resp} ->
99
-          {:halt, {:ok, resp}}
100
-
101
-        {:error, resp} ->
102
-          {:halt, {:error, resp}}
103
-
104
-        :fatal ->
105
-          {:cont, acc}
106
-      end
107
-    end)
108
-  end
109
-
110
-  defp do_request_potion(args) do
111
-    headers = ["user-agent": @user_agent]
112
-
113
-    options = [
114
-      timeout: timeout(),
115
-      follow_redirects: true,
116
-      auto_sni: true,
117
-      headers: headers
118
-    ]
119
-
120
-    default_args = [method: :get, headers: headers] ++ options
121
-    final_args = Keyword.merge(default_args, args)
122
-
123
-    try do
124
-      result =
125
-        HTTPotion.request(
126
-          final_args[:method],
127
-          final_args[:url],
128
-          Keyword.take(
129
-            final_args,
130
-            ~w(body headers query timeout follow_redirects basic_auth stream_to direct ibrowse auto_sni)a
131
-          )
132
-        )
133
-
134
-      case result do
135
-        %HTTPotion.Response{status_code: code, body: body} = resp ->
136
-          {:ok, %Koype.Http.Response{code: code, body: body, headers: resp.headers.hdrs, raw: resp}}
137
-
138
-        %HTTPotion.ErrorResponse{message: reason} = resp ->
139
-          {:error, %Koype.Http.Error{reason: reason, raw: resp, url: final_args[:url]}}
140
-      end
141
-    rescue
142
-      err ->
143
-        Logger.error("Failed to handle #{final_args[:url]} with HTTPotion: #{inspect(err)}")
144
-        :fatal
145
-    catch
146
-      err ->
147
-        Logger.error("Failed to handle #{final_args[:url]} with HTTPotion: #{err}")
148
-        :fatal
149
-    end
150
-  end
151
-
152
-  defp do_request_poison(args) do
153
-    headers = [{"user-agent", @user_agent}]
154
-    options = [ssl: [{:versions, [:"tlsv1.2"]}], recv_timeout: timeout(), follow_redirect: true, max_redirect: 10]
155
-    default_args = [method: :get, options: options, headers: headers, body: ""]
156
-    final_args = Keyword.merge(default_args, args)
157
-
158
-    try do
159
-      result =
160
-        HTTPoison.request(
161
-          final_args[:method],
162
-          final_args[:url],
163
-          final_args[:body],
164
-          final_args[:headers],
165
-          final_args[:options]
166
-        )
167
-
168
-      case result do
169
-        {:ok, %HTTPoison.Response{status_code: code, body: body} = resp} ->
170
-          {:ok, %Koype.Http.Response{code: code, body: body, headers: resp.headers |> Map.new(), raw: resp}}
171
-
172
-        {:error, %HTTPoison.Error{reason: reason} = resp} ->
173
-          {:error, %Koype.Http.Error{reason: reason, raw: resp, url: final_args[:url]}}
174
-      end
175
-    rescue
176
-      err ->
177
-        Logger.error("Failed to handle #{final_args[:url]} with HTTPoison: #{inspect(err)}")
178
-        :fatal
179
-    catch
180
-      err ->
181
-        Logger.error("Failed to handle #{final_args[:url]} with HTTPoison: #{err}")
182
-        :fatal
61
+  def request(method, url, opts) when is_binary(url) do
62
+    request_opts =
63
+      [
64
+        timeout: Keyword.get(opts, :timeout, Koype.Http.timeout()),
65
+        headers: Keyword.get(opts, :headers, %{}) |> Map.put("user-agent", @user_agent) |> Map.to_list() || nil,
66
+        body: Keyword.get(opts, :body, %{}) |> URI.encode_query() || nil,
67
+        query: Keyword.get(opts, :query, nil)
68
+      ]
69
+      |> Enum.reject(fn {_, v} -> is_nil(v) end)
70
+      |> Keyword.new()
71
+
72
+    case IndieWeb.Http.request(url, method, request_opts) do
73
+      {:ok, resp} ->
74
+        {:ok, %Koype.Http.Response{url: resp.url, code: resp.code, headers: resp.headers, body: resp.body, raw: resp}}
75
+
76
+      {:error, resp} ->
77
+        {:error, %Koype.Http.Error{reason: resp.message, raw: resp}}
183 78
     end
184 79
   end
185 80
 end

+ 4
- 0
lib/indieweb/mf2/remote.ex View File

@@ -86,4 +86,8 @@ defmodule IndieWeb.MF2.Remote do
86 86
   def flush(uri) do
87 87
     Cache.clear_for(uri, &key_func/1)
88 88
   end
89
+
90
+  def store(url, mf2_json) do
91
+    Cache.set(key_func(url), mf2_json)
92
+  end
89 93
 end

+ 5
- 2
lib/indieweb/micropub/entry.ex View File

@@ -432,9 +432,12 @@ defmodule IndieWeb.Micropub.Entry do
432 432
 
433 433
   defp do_extract_micro_mf2_for_uri(uri, author_data) do
434 434
     with(
435
-      {:ok, mf2} <- IndieWeb.MF2.Remote.fetch(uri),
435
+      {:ok, mf2} <- Apex.ap(Koype.Page.fetch(uri)),
436 436
       title <- Koype.Page.title(uri),
437
-      page_mf2 <- Enum.find(mf2["items"], {:error, :no_mf2}, fn item -> "h-card" not in item["type"] end)
437
+      page_mf2 <-
438
+        Enum.find(Map.take(mf2, ["items", :items]) |> Map.values() |> List.flatten(), {:error, :no_mf2}, fn item ->
439
+          "h-card" not in (Map.take(item, ["type", :type]) |> Map.values() |> List.flatten())
440
+        end)
438 441
     ) do
439 442
       props =
440 443
         [

+ 31
- 13
lib/page.ex View File

@@ -18,9 +18,27 @@
18 18
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
19 19
 defmodule Koype.Page do
20 20
   alias Koype.Http
21
-  defstruct title: nil, url: nil, type: nil
22
-  @enforce_keys ~w(title url type)a
23
-  @type t :: %__MODULE__{title: String.t(), url: URI.t(), type: binary()}
21
+
22
+  defmodule Structure do
23
+    defstruct name: nil, url: nil, content: nil, summary: nil, author: nil, media: %{}, categories: [], raw: nil
24
+    @enforce_keys ~w(name url content)a
25
+    @type t :: %__MODULE__{
26
+            name: String.t(),
27
+            url: URI.t(),
28
+            content: String.t(),
29
+            summary: String.t() | nil,
30
+            author: any() | nil,
31
+            raw: any() | nil,
32
+            media: list(),
33
+            categories: list()
34
+          }
35
+
36
+    @spec valid?(t()) :: boolean()
37
+    def valid?(structure) do
38
+      structure_keys = Map.keys(structure)
39
+      Enum.all?(~w(name url content)a, &Enum.member?(structure_keys, &1))
40
+    end
41
+  end
24 42
 
25 43
   @blank_image "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
26 44
 
@@ -66,20 +84,20 @@ defmodule Koype.Page do
66 84
     end
67 85
   end
68 86
 
87
+  # TODO: Handle the case of sites not supporting 'HEAD'
69 88
   def fetch(url) do
70
-    with({:ok, %Koype.Http.Response{body: page_html, code: code}} when code < 400 <- Koype.Http.get(url),
71
-      {:ok, content_title} <- Koype.Page.Fetcher.get_title(page_html),
72
-      {:ok, content_body} <- Koype.Page.Fetcher.get_body(page_html)
89
+    with(
90
+      {:ok, %Koype.Http.Response{code: code} = resp} when code < 400 <- Koype.Http.head(url),
91
+      {:ok, parsed_page} <- Koype.Page.Parser.resolve(url)
73 92
     ) do
74
-      content = %{"content" => %{"html" => content_body["html"], "value" => content_body["value"]}}
75
-
76
-      content
77
-      |> Koype.Page.Fetcher.set_author_to_mf2(page_html)
78
-      |> Koype.Page.Fetcher.set_media_to_mf2(page_html)
79
-      |> Koype.Page.Cacher.set(url)
93
+      mf2_json = Koype.Page.Parser.convert_to_mf2(parsed_page)
94
+      {:ok, mf2_json}
80 95
     else
96
+      {:ok, %Koype.Http.Response{} = resp} ->
97
+        {:error, :failed_to_check_page, reason: :status_code_not_handled, resp: resp}
98
+
81 99
       {:error, error} ->
82
-        {:error, :failed_to_normalize_page, error: error}
100
+        {:error, :failed_to_fetch_page, reason: error}
83 101
     end
84 102
   end
85 103
 end

+ 0
- 5
lib/page/cacher.ex View File

@@ -1,5 +0,0 @@
1
-defmodule Koype.Page.Cacher do
2
-  def set(mf2_json, url) do
3
-
4
-  end
5
-end

+ 0
- 6
lib/page/fetcher.ex View File

@@ -1,6 +0,0 @@
1
-defmodule Koype.Page.Fetcher do
2
-  def get_title(page_html)
3
-  def get_body(page_html)
4
-  def set_author_to_mf2(mf2_json, page_html)
5
-  def set_media_to_mf2(mf2_json, page_html)
6
-end

+ 64
- 0
lib/page/parser.ex View File

@@ -0,0 +1,64 @@
1
+defmodule Koype.Page.Parser do
2
+  @moduledoc """
3
+  Handles logic around parsing pages.
4
+
5
+  This module provides the basis of both determining if a parser will work for
6
+  a page and if useful information can be pulled from it. If it works for a page
7
+  but no useful content can be extracted; we'd have to considered it a failed
8
+  attempt.
9
+
10
+  TODO: Consider using a 'folding' approach; fetching what you can from particular
11
+  formats.
12
+  """
13
+
14
+  @doc "Takes in a URI and parses the content into a structured format."
15
+  @callback parse(response :: Koype.Http.Response.t()) :: {:ok, Koype.Page.Structure.t()} | {:error, any()}
16
+
17
+  @doc "Determines if the page has the right kind of content to use with this parserq."
18
+  @callback resolves?(response :: Koype.Http.Response.t()) :: boolean()
19
+
20
+  @parsers [
21
+    Koype.Page.Microformats2Parser,
22
+    Koype.Page.ActivityStreams2Parser,
23
+    Koype.Page.JsonLinkedDataParser,
24
+    Koype.Page.OpenGraphParser,
25
+    Koype.Page.TwitterCardParser,
26
+    Koype.Page.SchemaOrgParser
27
+  ]
28
+
29
+  # @doc "Finds the optimal parser for the provided site."
30
+  # def get_parser_for_url(url) do
31
+  # end
32
+
33
+  @doc "Walk through the list of parsers."
34
+  def resolve(url) do
35
+    with({:ok, %Koype.Http.Response{} = resp} <- Koype.Http.get(url)) do
36
+      Enum.reduce_while(@parsers, {:error, :no_parser_worked, %{}}, fn parser, {_, _, acc} ->
37
+        case parser.parse(resp) do
38
+          {:ok, _} = resp -> {:halt, resp}
39
+          _ = error -> {:cont, {:error, :no_parser_worked, Map.put(acc, parser, error)}}
40
+        end
41
+      end)
42
+    end
43
+  end
44
+
45
+  # TODO: Detect h-cards.
46
+  # TODO: Detect h-event.
47
+  # TODO: Detect h-review.
48
+  # TODO: Add media if any found.
49
+  def convert_to_mf2(%Koype.Page.Structure{} = structure) do
50
+    %{
51
+      items: [
52
+        %{
53
+          type: ["h-entry"],
54
+          properties:
55
+            structure
56
+            |> Map.take(~w(name content author url)a)
57
+            |> Map.put(:category, structure.categories)
58
+            |> Enum.reject(fn {_, value} -> is_nil(value) end)
59
+            |> Map.new()
60
+        }
61
+      ]
62
+    }
63
+  end
64
+end

+ 81
- 0
lib/page/parser/activitystreams2.ex View File

@@ -0,0 +1,81 @@
1
+defmodule Koype.Page.ActivityStreams2Parser do
2
+  defp do_find_as2_uri_from_html(resp) do
3
+    Enum.find_value(Map.to_list(resp), fn
4
+      {:body, html} ->
5
+        case Floki.find(html, "link[type='application/activity+json'][rel=alternate]") do
6
+          [] -> false
7
+          elem when not is_nil(elem) -> Floki.attribute(elem, "href") |> List.first()
8
+          _ -> false
9
+        end
10
+
11
+      {:raw, raw} ->
12
+        raw
13
+        |> IndieWeb.Http.extract_link_header_values()
14
+        |> Map.get("Link")
15
+
16
+      _ ->
17
+        nil
18
+    end)
19
+  end
20
+
21
+  defp do_find_as2_uri_from_request(resp) do
22
+    case Koype.Http.get(resp.url, headers: %{"Accept" => "application/activity+json"}) do
23
+      {:ok, %Koype.Http.Response{url: url, code: code}} when code < 400 -> url
24
+      _ -> nil
25
+    end
26
+  end
27
+
28
+  @moduledoc false
29
+  defp do_find_as2_uri(resp) do
30
+    Enum.find_value(
31
+      [&do_find_as2_uri_from_html/1, &do_find_as2_uri_from_request/1],
32
+      fn method -> method.(resp) end
33
+    )
34
+  end
35
+
36
+  def resolves?(resp), do: is_binary(do_find_as2_uri(resp))
37
+
38
+  defp do_get_author(uri) do
39
+    case IndieWeb.HCard.resolve(uri) do
40
+      {:ok, author} ->
41
+        Map.merge(author, %{"type" => ["h-card"]})
42
+
43
+      _ ->
44
+        uri
45
+    end
46
+  end
47
+
48
+  defp do_rewrap_tags(tags) do
49
+    Enum.map(tags, fn tag ->
50
+      %{
51
+        "type" => ["h-card"],
52
+        "name" => tag["name"],
53
+        "url" => tag["url"]
54
+      }
55
+    end)
56
+  end
57
+
58
+  # All we need is 'attributedTo', 'content', 'published', 'tag', 'url'
59
+  # TODO: use 'attachments' to fill out 'photo'/'video'/'audio'
60
+  def parse(%Koype.Http.Response{body: body, headers: headers} = resp) do
61
+    with(
62
+      url when is_binary(url) <- do_find_as2_uri(resp),
63
+      {:ok, %Koype.Http.Response{body: body, code: code}} when code < 400 <-
64
+        Koype.Http.get(url, headers: %{"Accept" => "application/activity+json"}),
65
+      {:ok, as2_json} <- Jason.decode(body)
66
+    ) do
67
+      {:ok,
68
+       %Koype.Page.Structure{
69
+         url: as2_json["url"] || url,
70
+         categories: do_rewrap_tags(as2_json["tag"]),
71
+         author: do_get_author(as2_json["attributedTo"]),
72
+         summary: as2_json["summary"],
73
+         content: as2_json["content"],
74
+         raw: as2_json
75
+       }}
76
+    else
77
+      nil ->
78
+        {:error, :no_as2_url_found}
79
+    end
80
+  end
81
+end

+ 20
- 0
lib/page/parser/json_linked_data.ex View File

@@ -0,0 +1,20 @@
1
+defmodule Koype.Page.JsonLinkedDataParser do
2
+  def resolves?(resp) do
3
+    resp.body |> Floki.find("script[type=application/ld+json]") != nil
4
+  end
5
+
6
+  def parse(resp) do
7
+    if resolves?(resp) do
8
+      case resp.body |> Floki.find("script[type=application/ld+json]") |> Floki.text(js: true) do
9
+        nil ->
10
+          {:error, :no_json_linked_data}
11
+
12
+        text ->
13
+          Apex.ap(text)
14
+          nil
15
+      end
16
+    else
17
+      {:error, :no_json_linked_data}
18
+    end
19
+  end
20
+end

+ 39
- 0
lib/page/parser/microformats2.ex View File

@@ -0,0 +1,39 @@
1
+defmodule Koype.Page.Microformats2Parser do
2
+  @moduledoc false
3
+
4
+  # TODO: Check if a top-level `h-` element exists on the site.
5
+  # TODO: Move this checking logic into the IndieWeb library.
6
+  def resolves?(%Koype.Http.Response{body: body}) do
7
+    cond do
8
+      Floki.find(body, ".h-entry") != [] -> true
9
+      Floki.find(body, ".h-event") != [] -> true
10
+      Floki.find(body, ".h-card") != [] -> true
11
+      Floki.find(body, ".h-feed") != [] -> true
12
+      body == "" -> false
13
+      true -> false
14
+    end
15
+  end
16
+
17
+  def parse(%Koype.Http.Response{body: body, url: url}) do
18
+    with(
19
+      mf2 when is_map(mf2) <- IndieWeb.MF2.parse(body, url),
20
+      {:ok, entry_mf2} <- IndieWeb.MF2.get_format(mf2, :entry),
21
+      {:ok, author} <- IndieWeb.HCard.resolve(url)
22
+    ) do
23
+      {:ok,
24
+       %Koype.Page.Structure{
25
+         name: entry_mf2[:name],
26
+         url: url,
27
+         content: entry_mf2[:content],
28
+         summary: entry_mf2[:summary],
29
+         author: author,
30
+         media: [photo: entry_mf2[:photo], video: entry_mf2[:video], audio: entry_mf2[:audio]],
31
+         categories: entry_mf2[:category],
32
+         raw: mf2
33
+       }}
34
+    else
35
+      _ ->
36
+        {:error, :failed_to_get_useful_mf2}
37
+    end
38
+  end
39
+end

+ 23
- 0
lib/page/parser/open_graph.ex View File

@@ -0,0 +1,23 @@
1
+defmodule Koype.Page.OpenGraphParser do
2
+  def resolves?(resp) do
3
+    resp.body |> Floki.find("meta[property=og:url]") != nil
4
+  end
5
+
6
+  def parse(resp) do
7
+    if resolves?(resp) do
8
+      og_data = OpenGraph.parse(resp.body)
9
+
10
+      {:ok,
11
+       %Koype.Page.Structure{
12
+         url: og_data.url,
13
+         categories: [],
14
+         summary: og_data.description,
15
+         content: og_data.description,
16
+         name: og_data.title,
17
+         raw: og_data
18
+       }}
19
+    else
20
+      {:error, :no_open_graph_data}
21
+    end
22
+  end
23
+end

+ 5
- 7
lib/repo/webmention.ex View File

@@ -83,11 +83,8 @@ defmodule Koype.Repo.Webmention do
83 83
     |> validate_exclusion(:source, [attrs[:target]])
84 84
   end
85 85
 
86
-  def update_author(model) do
87
-    case IndieWeb.HCard.resolve(model.source) do
88
-      {:ok, author} -> changeset(model, %{author: author["url"]}) |> Koype.Repo.update()
89
-      _ -> {:ok, model}
90
-    end
86
+  def update_author(model, author) do
87
+    changeset(model, %{author: author["url"]}) |> Koype.Repo.update()
91 88
   end
92 89
 
93 90
   def create(args) do
@@ -99,8 +96,9 @@ defmodule Koype.Repo.Webmention do
99 96
           type: args[:type]
100 97
         }),
101 98
       {:ok, record} <- Koype.Repo.insert(cs),
102
-      {:ok, record} <- update_author(record),
103
-      {:ok, _path} <- __MODULE__.Json.persist(record, author: IndieWeb.HCard.resolve(record.author), mf2: args[:mf2])
99
+      {:ok, author} <- IndieWeb.HCard.resolve(args[:source]),
100
+      {:ok, record} <- update_author(record, author),
101
+      {:ok, _path} <- __MODULE__.Json.persist(record, author: author, mf2: args[:mf2])
104 102
     ) do
105 103
       Logger.info("Saved a new Webmention from #{args[:source]} to #{args[:target]}")
106 104
 

+ 2
- 0
lib/storage.ex View File

@@ -52,6 +52,8 @@ defmodule Koype.Storage do
52 52
         end
53 53
       end)
54 54
 
55
+  def will_trigger_upload?(_), do: false
56
+
55 57
   def does_property_need_upload?(key) when is_binary(key),
56 58
     do: Enum.member?(@uploadable_keys, key)
57 59
 

+ 3
- 2
lib/webmention.ex View File

@@ -182,7 +182,8 @@ defmodule Koype.Webmention do
182 182
 
183 183
     with(
184 184
       {:ok, mf2} when is_map(mf2) <- IndieWeb.MF2.Remote.fetch(source),
185
-      entry_mf2 when is_map(entry_mf2) <- IndieWeb.MF2.get_format(mf2, "entry")
185
+      entry_mf2 when is_map(entry_mf2) <- IndieWeb.MF2.get_format(mf2, "entry"),
186
+      {:ok, author} <- IndieWeb.HCard.resolve(source)
186 187
     ) do
187 188
       types = IndieWeb.Post.extract_types(entry_mf2["properties"])
188 189
       type = IndieWeb.Post.determine_type(entry_mf2["properties"], types)
@@ -196,7 +197,7 @@ defmodule Koype.Webmention do
196 197
 
197 198
       case Koype.Repo.upsert(Koype.Repo.Webmention, params) do
198 199
         {:ok, record} = result ->
199
-          record |> Koype.Repo.Webmention.update_author()
200
+          record |> Koype.Repo.Webmention.update_author(author)
200 201
 
201 202
         {:error, error} = err ->
202 203
           Logger.error("Failed to create Webmention from #{source} to #{target_url}: #{inspect(error)}.")

+ 9
- 5
mix.exs View File

@@ -78,7 +78,7 @@ defmodule Koype.Mixfile do
78 78
         :logster,
79 79
         :que,
80 80
         :liquid,
81
-        :furlex
81
+        :readability
82 82
       ]
83 83
     ]
84 84
   end
@@ -105,7 +105,10 @@ defmodule Koype.Mixfile do
105 105
       {:ecto, "~> 2.2.0"},
106 106
       {:ex_aws, "~> 2.1"},
107 107
       {:ex_aws_s3, "~> 2.0"},
108
-      {:ex_doc, ">= 0.19.0", only: :dev},
108
+      {:ex_cldr, "~> 2.0"},
109
+      {:ex_cldr_numbers, "~> 2.1"},
110
+      {:ex_cldr_territories, "~> 2.0"},
111
+      {:ex_doc, ">= 0.19.0"},
109 112
       {:ex_machina, "~> 2.2", only: :test},
110 113
       {:ex_url, "~> 1.0.0"},
111 114
       {:excoveralls, "~> 0.8", only: :test},
@@ -113,7 +116,6 @@ defmodule Koype.Mixfile do
113 116
       {:exvcr, "~> 0.10", only: :test},
114 117
       {:faker, "~> 0.12.0", only: :test},
115 118
       {:floki, "~> 0.20.0", override: true},
116
-      {:furlex, "~> 0.3.3"},
117 119
       {:gettext, "~> 0.11"},
118 120
       {:git_hooks, "~> 0.2.0", only: :dev},
119 121
       {:guardian, "~> 1.0"},
@@ -122,8 +124,8 @@ defmodule Koype.Mixfile do
122 124
       {:hound, "~> 1.0", only: :test},
123 125
       {:html_sanitize_ex, "~> 1.3.0-rc3"},
124 126
       {:httpoison, "~> 1.3.0", override: true},
125
-      {:inch_ex, github: "rrrene/inch_ex"},
126
-      {:indieweb, "~> 0.0.0"},
127
+      {:inch_ex, git: "https://github.com/rrrene/inch_ex"},
128
+      {:indieweb, git: "https://git.jacky.wtf/indieweb/elixir", branch: "develop"},
127 129
       {:inflex, "~> 1.10.0", override: true},
128 130
       {:jason, "~> 1.1.0"},
129 131
       {:liquid, git: "https://github.com/bettyblocks/liquid-elixir", commit: " 1a29cbc"},
@@ -131,6 +133,7 @@ defmodule Koype.Mixfile do
131 133
       {:microformats2, "~> 0.2.1"},
132 134
       {:mime, "~> 1.3.0"},
133 135
       {:mock, "~> 0.3.0", only: :test},
136
+      {:open_graph, "~> 0.0.2"},
134 137
       {:page_object, "~> 0.4.0", only: :test},
135 138
       {:phoenix, "~> 1.4.2"},
136 139
       {:phoenix_ecto, "~> 3.2"},
@@ -142,6 +145,7 @@ defmodule Koype.Mixfile do
142 145
       {:plug_cowboy, "~> 2.0"},
143 146
       {:pretty_print_formatter, "~> 0.1.5", only: :dev},
144 147
       {:que, "~> 0.8.0"},
148
+      {:readability, "~> 0.10.0"},
145 149
       {:scrivener_list, "~> 1.0"},
146 150
       {:scrivener_ecto, "~> 1.0"},
147 151
       {:scrivener_headers, "~> 3.1"},

+ 11
- 8
mix.lock View File

@@ -11,7 +11,7 @@
11 11
   "cachex": {:hex, :cachex, "3.1.3", "86ed0669ea4b2f3e3982dbb5c6ca9e0964e46738e572c9156f22ceb75f57c336", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
12 12
   "calendar": {:hex, :calendar, "0.17.4", "22c5e8d98a4db9494396e5727108dffb820ee0d18fed4b0aa8ab76e4f5bc32f1", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
13 13
   "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
14
-  "cldr_utils": {:hex, :cldr_utils, "2.0.5", "eba0f4cc86861b74f2c1180fe7f6fa25f9e9a3b06365fa7468213c9ec3fd392c", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
14
+  "cldr_utils": {:hex, :cldr_utils, "2.2.0", "4f6ac090fc1871037ac41265c98bf0f785b2a4ede144dd2a5282986a5311dd14", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
15 15
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"},
16 16
   "confex": {:hex, :confex, "3.4.0", "8b1c3cc7a93320291abb31223a178df19d7f722ee816c05a8070c8c9a054560d", [:mix], [], "hexpm"},
17 17
   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"},
@@ -21,7 +21,7 @@
21 21
   "credo": {:hex, :credo, "0.10.2", "03ad3a1eff79a16664ed42fc2975b5e5d0ce243d69318060c626c34720a49512", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
22 22
   "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
23 23
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
24
-  "dialyxir": {:hex, :dialyxir, "1.0.0-rc.4", "71b42f5ee1b7628f3e3a6565f4617dfb02d127a0499ab3e72750455e986df001", [:mix], [{:erlex, "~> 0.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"},
24
+  "dialyxir": {:hex, :dialyxir, "1.0.0-rc.5", "c9c2379c59cf2dfc74690f48866e33ffb55ff660e5e02405c14614d204efdc4f", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"},
25 25
   "double": {:hex, :double, "0.6.6", "2280fa0800dd582ec1c542bc3aba134950bebf9281e847ecf8f50ddf19da830c", [:mix], [], "hexpm"},
26 26
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
27 27
   "ecto": {:hex, :ecto, "2.2.9", "031d55df9bb430cb118e6f3026a87408d9ce9638737bda3871e5d727a3594aae", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
@@ -31,9 +31,9 @@
31 31
   "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"},
32 32
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
33 33
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},
34
-  "ex_cldr": {:hex, :ex_cldr, "2.3.2", "4ca867a5b763fef88e4155f088d69e73d611ca4685d7e8766502668b1a3ed0a6", [:mix], [{:cldr_utils, "~> 2.0", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.13", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
35
-  "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.2.3", "54fec53ce36c420c863e6e3fdbbe5798e63669aeb6474cc6901cc028ff03066f", [:mix], [{:ex_cldr, "~> 2.0", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
36
-  "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.4.1", "487a45fed201d2271b292ae801321e06fec208eab6b73f257810451e391889ae", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.1", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.2", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
34
+  "ex_cldr": {:hex, :ex_cldr, "2.5.0", "07e01e7abfe33a832a12d67a3d0658f63f9b0bf73e819741e952064c545374af", [:mix], [{:cldr_utils, "~> 2.1", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.13", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
35
+  "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.2.5", "2d0dcc30be45ec4468f01362b724bc9a459679ed2050817121e736710394f875", [:mix], [{:ex_cldr, "~> 2.4", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
36
+  "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.5.0", "1cbba0e0253a35840ce2bc5983b75bf72d1df43b9cfe7451bd10e1ed42e0fbdb", [:mix], [{:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.5", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.2", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
37 37
   "ex_cldr_territories": {:hex, :ex_cldr_territories, "2.0.0", "e1a56ab2904ab6cff2a270dbe8222927ee2ccfe746a9965142bd7a589a5c50f0", [:mix], [{:ex_cldr, "~> 2.0", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
38 38
   "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
39 39
   "ex_image_info": {:hex, :ex_image_info, "0.2.4", "610002acba43520a9b1cf1421d55812bde5b8a8aeaf1fe7b1f8823e84e762adb", [:mix], [], "hexpm"},
@@ -64,8 +64,8 @@
64 64
   "httpotion": {:hex, :httpotion, "3.1.1", "b8ad199dea2c56a70c89e7f9e4d09898c7e85871783b7417c04cb4f1d4d8e919", [:mix], [{:ibrowse, "== 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: false]}], "hexpm"},
65 65
   "ibrowse": {:hex, :ibrowse, "4.4.0", "2d923325efe0d2cb09b9c6a047b2835a5eda69d8a47ed6ff8bc03628b764e991", [:rebar3], [], "hexpm"},
66 66
   "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
67
-  "inch_ex": {:git, "https://github.com/rrrene/inch_ex.git", "45ecbc4b57fa8b943b9e4cd0d4da72da95c9dd4a", []},
68
-  "indieweb": {:hex, :indieweb, "0.0.42", "8e21fda9dd30b2c57517f841c7b725e03c41abf18f7e4f9885a6110c5289f0a0", [:mix], [{:cachex, "~> 3.1.0", [hex: :cachex, repo: "hexpm", optional: false]}, {:microformats2, "~> 0.2.0", [hex: :microformats2, repo: "hexpm", optional: false]}], "hexpm"},
67
+  "inch_ex": {:git, "https://github.com/rrrene/inch_ex", "45ecbc4b57fa8b943b9e4cd0d4da72da95c9dd4a", []},
68
+  "indieweb": {:git, "https://git.jacky.wtf/indieweb/elixir", "7179916d06a68d8413c83c9944873dedbcf3a5bd", [branch: "develop"]},
69 69
   "inflex": {:hex, :inflex, "1.10.0", "8366a7696e70e1813aca102e61274addf85d99f4a072b2f9c7984054ea1b9d29", [:mix], [], "hexpm"},
70 70
   "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
71 71
   "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"},
@@ -103,6 +103,7 @@
103 103
   "pretty_print_formatter": {:hex, :pretty_print_formatter, "0.1.5", "e1aa7f0e03c5b4fc33c32083beea4d7ece1070e9250a2e9e692cead6940f2eaf", [:mix], [], "hexpm"},
104 104
   "que": {:hex, :que, "0.8.0", "873d46b86966c16776ee9c5482597d71c83a49dc9a5ea9a77e6095218a74ece0", [:mix], [{:ex_utils, "~> 0.1.6", [hex: :ex_utils, repo: "hexpm", optional: false]}, {:memento, "~> 0.2.1", [hex: :memento, repo: "hexpm", optional: false]}], "hexpm"},
105 105
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
106
+  "readability": {:hex, :readability, "0.10.0", "934212018e70346a982927ee4b32d3ddb3d5feba7bf7ab04f57da66ced5ab7a2", [:mix], [{:floki, "~> 0.20", [hex: :floki, repo: "hexpm", optional: false]}, {:httpoison, "~> 0.13.0", [hex: :httpoison, repo: "hexpm", optional: false]}], "hexpm"},
106 107
   "recase": {:hex, :recase, "0.4.0", "8fb52846f75948156385af2dfdc12f69e5ce27b022a9d1682c70a2fb3ed149c7", [:mix], [], "hexpm"},
107 108
   "redix": {:hex, :redix, "0.8.2", "c25158f905bcf8842e9a11411d65b9257ac70057c4330521d1a4d2a44b4f7ecf", [:mix], [], "hexpm"},
108 109
   "sbroker": {:hex, :sbroker, "1.0.0", "28ff1b5e58887c5098539f236307b36fe1d3edaa2acff9d6a3d17c2dcafebbd0", [:rebar3], [], "hexpm"},
@@ -119,10 +120,12 @@
119 120
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
120 121
   "stream_gzip": {:hex, :stream_gzip, "0.3.1", "367ef2b91920cd6b906eeda78287a4da5330edf6e4147bbe1fc314802e0208bc", [:mix], [], "hexpm"},
121 122
   "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
123
+  "tesla": {:git, "https://github.com/jalcine/tesla", "390a267a84d3b535f12e5efb6f58d66f157de57b", [branch: "jalcine/check-regex-run-results"]},
124
+  "tesla_request_id": {:hex, :tesla_request_id, "0.2.0", "9745364ed3c850864fb2744daefb9b7104fe7f1efcf34eb350c61da805de2a46", [:mix], [{:tesla, "~> 1.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"},
122 125
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
123 126
   "toniq": {:hex, :toniq, "1.2.3", "e80ecafb0aaf131d74cdb7b39bb4de263585b012c59d263ec9f2769df30398a5", [:mix], [{:exredis, ">= 0.1.1", [hex: :exredis, repo: "hexpm", optional: false]}, {:uuid, "~> 1.0", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm"},
124 127
   "totpex": {:hex, :totpex, "0.1.3", "ae022eab70e8e230a0a65300b0d0dd67f510e0b7243c0d608a827c7b22ca6b51", [:mix], [], "hexpm"},
125
-  "tzdata": {:hex, :tzdata, "0.5.19", "7962a3997bf06303b7d1772988ede22260f3dae1bf897408ebdac2b4435f4e6a", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
128
+  "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
126 129
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
127 130
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"},
128 131
   "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"},

+ 364
- 478
package-lock.json
File diff suppressed because it is too large
View File


+ 2
- 2
package.json View File

@@ -18,11 +18,11 @@
18 18
     "autoprefixer": "9.3.1",
19 19
     "axios": "0.18.0",
20 20
     "css-loader": "2.1.0",
21
-    "cssnano": "4.1.7",
21
+    "cssnano": "4.1.10",
22 22
     "feather-icons": "4.19.0",
23 23
     "ladda": "2.0.1",
24 24
     "leaflet": "1.3.4",
25
-    "parcel": "1.12.0",
25
+    "parcel": "1.12.3",
26 26
     "phoenix": "file:deps/phoenix",
27 27
     "phoenix_html": "file:deps/phoenix_html",
28 28
     "pixrem": "4.0.1",

+ 57
- 0
test/fixtures/vcr_cassettes/page_fetch_indieweb.json
File diff suppressed because it is too large
View File


+ 63
- 0
test/fixtures/vcr_cassettes/page_fetch_success.json
File diff suppressed because it is too large
View File


+ 45
- 0
test/fixtures/vcr_cassettes/page_fetch_unreachable.json View File

@@ -0,0 +1,45 @@
1
+[
2
+  {
3
+    "request": {
4
+      "body": "",
5
+      "headers": {
6
+        "user-agent": "Koype/0.0.7 (https://koype.net/faq/user-agent)"
7
+      },
8
+      "method": "get",
9
+      "options": {
10
+        "ssl_options": {
11
+          "server_name_indication": [
12
+            104,
13
+            116,
14
+            116,
15
+            112,
16
+            98,
17
+            105,
18
+            110,
19
+            46,
20
+            111,
21
+            114,
22
+            103
23
+          ]
24
+        }
25
+      },
26
+      "request_body": "",
27
+      "url": "https://httpbin.org/status/500"
28
+    },
29
+    "response": {
30
+      "binary": false,
31
+      "body": "",
32
+      "headers": {
33
+        "Access-Control-Allow-Credentials": "true",
34
+        "Access-Control-Allow-Origin": "*",
35
+        "Content-Type": "text/html; charset=utf-8",
36
+        "Date": "Mon, 25 Mar 2019 17:49:05 GMT",
37
+        "Server": "nginx",
38
+        "Content-Length": "0",
39
+        "Connection": "keep-alive"
40
+      },
41
+      "status_code": 500,
42
+      "type": "ok"
43
+    }
44
+  }
45
+]

+ 82
- 0
test/fixtures/vcr_cassettes/parse_activtystreams_parse_success.json
File diff suppressed because it is too large
View File


+ 1
- 1
test/integration/controllers/contact_controller_test.exs View File

@@ -1,7 +1,7 @@
1 1
 defmodule Koype.Web.ContactControllerTest do
2 2
   use Koype.Web.ConnCase
3 3
   use Koype.DataCase
4
-  use ExVCR.Mock
4
+  use Koype.HttpMock
5 5
   import Koype.Factory
6 6
 
7 7
   describe "GET .index/2" do

+ 7
- 0
test/support/http.ex View File

@@ -0,0 +1,7 @@
1
+defmodule Koype.HttpMock do
2
+  defmacro __using__(_) do
3
+    quote do
4
+      use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney
5
+    end
6
+  end
7
+end

+ 1
- 1
test/support/steps/form.ex View File

@@ -2,7 +2,7 @@ defmodule Koype.Feature.Steps.Shared.Form do
2 2
   defmacro __using__(_) do
3 3
     quote do
4 4
       alias Koype.Test.Pages.Form
5
-      use ExVCR.Mock
5
+      use Koype.HttpMock
6 6
 
7 7
       defand ~r/^I enter an? (?<file>.+)( file)? for upload to "(?<field>[^"]+)"$/,
8 8
              %{field: field, file: file_type},

+ 1
- 1
test/unit/contact_test.exs View File

@@ -2,7 +2,7 @@ defmodule Koype.ContactTest do
2 2
   use Koype.Test.BaseCase
3 3
   use Koype.DataCase
4 4
   import Koype.Factory
5
-  use ExVCR.Mock
5
+  use Koype.HttpMock
6 6
   alias Koype.Contact, as: Subject
7 7
 
8 8
   describe ".find/1" do

+ 1
- 76
test/unit/http_test.exs View File

@@ -4,82 +4,7 @@ defmodule Koype.HttpTest do
4 4
   import ExUnit.CaptureLog
5 5
   import Mock
6 6
 
7
-  use ExVCR.Mock
7
+  use Koype.HttpMock
8 8
 
9 9
   @url "https://indieweb.org"
10
-
11
-  describe ".request/2" do
12
-    test "successfully fetches a page" do
13
-      use_cassette "http_fetch_page_success" do
14
-        with_mock(HTTPotion, [:passthrough], []) do
15
-          assert {:ok, _} = Subject.request(:get, @url)
16
-          assert_called(HTTPotion.request(:_, :_, :_))
17
-        end
18
-      end
19
-    end
20
-
21
-    test "uses HTTPoison if HTTPotion throws error" do
22
-      use_cassette "http_fetch_page_success" do
23
-        with_mocks([
24
-          {HTTPotion, [], request: fn _, _, _ -> throw(:foo) end},
25
-          {HTTPoison, [:passthrough], []}
26
-        ]) do
27
-          assert capture_log(fn ->
28
-                   assert {:ok, _} = Subject.request(:get, @url)
29
-                 end) =~ ~r/Failed to handle .* with HTTPotion/
30
-
31
-          assert_called(HTTPotion.request(:_, :_, :_))
32
-          assert_called(HTTPoison.request(:_, :_, :_, :_, :_))
33
-        end
34
-      end
35
-    end
36
-
37
-    test "uses HTTPoison if HTTPotion raises exception" do
38
-      use_cassette "http_fetch_page_success" do
39
-        with_mocks([
40
-          {HTTPotion, [], request: fn _, _, _ -> Not.A.Real.function() end},
41
-          {HTTPoison, [:passthrough], []}
42
-        ]) do
43
-          assert capture_log(fn ->
44
-                   assert {:ok, _} = Subject.request(:get, @url)
45
-                 end) =~ ~r/Failed to handle .* with HTTPotion/
46
-
47
-          assert_called(HTTPotion.request(:_, :_, :_))
48
-          assert_called(HTTPoison.request(:_, :_, :_, :_, :_))
49
-        end
50
-      end
51
-    end
52
-
53
-    test "hard fails if nothing works (error and exception)" do
54
-      use_cassette "http_fetch_page_success" do
55
-        with_mocks([
56
-          {HTTPotion, [], request: fn _, _, _ -> throw(:foo) end},
57
-          {HTTPoison, [], request: fn _, _, _, _, _ -> Not.A.Real.function() end}
58
-        ]) do
59
-          assert capture_log(fn ->
60
-                   assert {:error, _} = Subject.request(:get, @url)
61
-                 end) =~ ~r/Failed to handle .* with HTTPoison/
62
-
63
-          assert_called(HTTPotion.request(:_, :_, :_))
64
-          assert_called(HTTPoison.request(:_, :_, :_, :_, :_))
65
-        end
66
-      end
67
-    end
68
-
69
-    test "hard fails if nothing works (both errors)" do
70
-      use_cassette "http_fetch_page_success" do
71
-        with_mocks([
72
-          {HTTPotion, [], request: fn _, _, _ -> throw(:foo) end},
73
-          {HTTPoison, [], request: fn _, _, _, _, _ -> throw(:bar) end}
74
-        ]) do
75
-          assert capture_log(fn ->
76
-                   assert {:error, _} = Subject.request(:get, @url)
77
-                 end) =~ ~r/Failed to handle .* with HTTPoison/
78
-
79
-          assert_called(HTTPotion.request(:_, :_, :_))
80
-          assert_called(HTTPoison.request(:_, :_, :_, :_, :_))
81
-        end
82
-      end
83
-    end
84
-  end
85 10
 end

+ 1
- 1
test/unit/indieweb/mf2/remote_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.MF2.RemoteTest do
2 2
   use Koype.Test.BaseCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   alias IndieWeb.MF2.Remote
5 5
   alias Koype.Cache
6 6
   import Mock

+ 1
- 1
test/unit/indieweb/micropub/content_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.Micropub.ContentTest do
2 2
   use Koype.DataCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   alias IndieWeb.Micropub.Content
5 5
   doctest Content
6 6
 

+ 1
- 1
test/unit/indieweb/micropub/entry_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.Micropub.EntryTest do
2 2
   use Koype.DataCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   import Mock
5 5
   import Koype.Factory
6 6
 

+ 1
- 1
test/unit/indieweb/micropub_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.MicropubTest do
2 2
   use Koype.Test.BaseCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   alias IndieWeb.Micropub, as: Subject
5 5
 
6 6
   describe ".reserved_keyword?/1" do

+ 1
- 1
test/unit/indieweb/relme_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.SubjectTest do
2 2
   use Koype.DataCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   alias IndieWeb.RelMe, as: Subject
5 5
   import Mock
6 6
   import Koype.Factory

+ 1
- 1
test/unit/indieweb/syndication_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule IndieWeb.SyndicationTest do
2 2
   use Koype.DataCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   import Koype.Factory
5 5
   alias IndieWeb.Syndication, as: Subject
6 6
   doctest Subject

+ 60
- 0
test/unit/page/parser/activitystreams2_test.exs View File

@@ -0,0 +1,60 @@
1
+defmodule Koype.Page.ActivityStreams2ParserTest do
2
+  use Koype.Test.BaseCase, async: true
3
+  use Koype.HttpMock
4
+  alias Koype.Page.ActivityStreams2Parser, as: Subject
5
+
6
+  describe ".resolves?/1" do
7
+    @url Faker.Internet.url()
8
+
9
+    test "true for pages that expose an AS2 link to the page" do
10
+      html = """
11
+      <html>
12
+      <head>
13
+      <link href='#{@url}/page.as2.json' rel='alternate' type='application/activity+json'>
14
+      </head>
15
+      """
16
+
17
+      assert Subject.resolves?(%Koype.Http.Response{
18
+               body: html,
19
+               url: @url,
20
+               raw: nil
21
+             })
22
+    end
23
+
24
+    test "true for pages that expose a AS2 link in the Link header" do
25
+      assert Subject.resolves?(%Koype.Http.Response{
26
+               url: @url,
27
+               raw: %IndieWeb.Http.Response{
28
+                 code: 200,
29
+                 body: "",
30
+                 url: @url,
31
+                 headers: %{},
32
+                 raw: %{
33
+                   opts: [
34
+                     rels: %{"Link" => "dfdsfsdf", "foo" => 333}
35
+                   ]
36
+                 }
37
+               }
38
+             })
39
+    end
40
+
41
+    test "false otherwise"
42
+  end
43
+
44
+  describe ".parse/1" do
45
+    test "gives a structured response from AS2 JSON" do
46
+      {:ok, resp} = Koype.Http.get("https://playvicious.social/@jalcine/101821423103972918")
47
+      formed_page_from_as2 = Subject.parse(resp)
48
+
49
+      assert {:ok,
50
+              %Koype.Page.Structure{
51
+                url: "https://playvicious.social/@jalcine/101821423103972918",
52
+                categories: [],
53
+                content: "<p>me right now to this code that just passed all of the prewritten tests</p>"
54
+              }} = formed_page_from_as2
55
+    end
56
+
57
+    @tag skip: true
58
+    test "fails if AS2 JSON isn't useful"
59
+  end
60
+end

+ 0
- 0
test/unit/page/parser/json_linked_data_parser_test.exs View File


+ 48
- 0
test/unit/page/parser/microformats2_test.exs View File

@@ -0,0 +1,48 @@
1
+defmodule Koype.Page.Microformats2ParserTest do
2
+  use Koype.Test.BaseCase, async: true
3
+  alias Koype.Page.Microformats2Parser, as: Subject
4
+
5
+  describe ".resolves?/1" do
6
+    test "true for pages with a 'h-' element" do
7
+      assert Subject.resolves?(%Koype.Http.Response{
8
+               body: "<html><body class='h-event'></body></html>",
9
+               url: Faker.Internet.url(),
10
+               raw: nil
11
+             })
12
+
13
+      assert Subject.resolves?(%Koype.Http.Response{
14
+               body: "<html><body class='h-feed'></body></html>",
15
+               url: Faker.Internet.url(),
16
+               raw: nil
17
+             })
18
+
19
+      assert Subject.resolves?(%Koype.Http.Response{
20
+               body: "<html><body class='h-entry'></body></html>",
21
+               url: Faker.Internet.url(),
22
+               raw: nil
23
+             })
24
+
25
+      assert Subject.resolves?(%Koype.Http.Response{
26
+               body: "<html><body class='h-card'></body></html>",
27
+               url: Faker.Internet.url(),
28
+               raw: nil
29
+             })
30
+    end
31
+
32
+    test "false for empty pages" do
33
+      refute Subject.resolves?(%Koype.Http.Response{
34
+               body: "",
35
+               url: Faker.Internet.url(),
36
+               raw: nil
37
+             })
38
+    end
39
+
40
+    test "false for pages without 'h-' elements" do
41
+      refute Subject.resolves?(%Koype.Http.Response{
42
+               body: "<html><body></body></html>",
43
+               url: Faker.Internet.url(),
44
+               raw: nil
45
+             })
46
+    end
47
+  end
48
+end

+ 0
- 0
test/unit/page/parser/schema_org_parser_test.exs View File


+ 1
- 0
test/unit/page/parser/twitter_card_parser_test.exs View File

@@ -0,0 +1 @@
1
+

+ 0
- 0
test/unit/page/parser_test.exs View File


+ 45
- 1
test/unit/page_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule Koype.PageTest do
2 2
   use Koype.Test.BaseCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   alias Koype.Page, as: Subject
5 5
 
6 6
   describe ".title/1" do
@@ -52,4 +52,48 @@ defmodule Koype.PageTest do
52 52
       end
53 53
     end
54 54
   end
55
+
56
+  describe ".fetch/1" do
57
+    @url "https://twitter.com/jackyalcine/status/1101331482067587077"
58
+
59
+    test "converts a Web page into an Microformats article" do
60
+      url = "https://www.marieclaire.com/celebrity/a26102917/lupita-nyongo-us-interview-2019/"
61
+
62
+      use_cassette "page_fetch_success" do
63
+        assert {:ok, %{"url" => url} = mf2_json} = Subject.fetch(url)
64
+
65
+        value = mf2_json["content"]["value"] |> Enum.join("\n")
66
+        assert value =~ "thin golden thread"
67
+
68
+        name = mf2_json["name"]
69
+        assert name =~ "She's Doing It Anyways"
70
+
71
+        refute mf2_json["author"]
72
+        refute mf2_json["photo"]
73
+        refute mf2_json["video"]
74
+        refute mf2_json["published_at"]
75
+        refute mf2_json["updated_at"]
76
+      end
77
+    end
78
+
79
+    test "fails if the title can't be pulled out" do
80
+      use_cassette "page_fetch_indieweb" do
81
+        assert {:error, :failed_to_normalize_page, reason: :missing_title} = Subject.fetch(@url)
82
+      end
83
+    end
84
+
85
+    test "fails if the content can't be pulled out" do
86
+      use_cassette "page_fetch_indieweb" do
87
+        assert {:error, :failed_to_normalize_page, reason: :missing_content} = Subject.fetch(@url)
88
+      end
89
+    end
90
+
91
+    test "fails if the page can't be reached" do
92
+      url = "https://httpbin.org/status/500"
93
+
94
+      use_cassette "page_fetch_unreachable" do
95
+        assert {:error, :failed_to_check_page, reason: :bad_status_code, resp: _} = Subject.fetch(url)
96
+      end
97
+    end
98
+  end
55 99
 end

+ 1
- 1
test/unit/repo/entry_test.exs View File

@@ -1,7 +1,7 @@
1 1
 defmodule Koype.Repo.EntryTest do
2 2
   use Koype.Web.ConnCase
3 3
   use Koype.DataCase
4
-  use ExVCR.Mock
4
+  use Koype.HttpMock
5 5
   import Koype.Factory
6 6
   alias Koype.Repo
7 7
   alias Koype.Repo.Entry, as: Subject

+ 1
- 1
test/unit/repo/webmention_test.exs View File

@@ -1,6 +1,6 @@
1 1
 defmodule Koype.Repo.WebmentionTest do
2 2
   use Koype.DataCase
3
-  use ExVCR.Mock
3
+  use Koype.HttpMock
4 4
   import Koype.Factory
5 5
   alias Koype.Repo.Webmention, as: Subject
6 6
 

+ 5
- 8
test/unit/storage/json_test.exs View File

@@ -1,8 +1,9 @@
1 1
 defmodule Koype.Storage.JsonTest do
2
-  use Koype.DataCase
3 2
   alias Koype.Storage.Json
4 3
   import Mock
5 4
   import Koype.Factory
5
+  use Koype.DataCase
6
+  use Koype.HttpMock
6 7
   doctest Json
7 8
 
8 9
   @model %Koype.Repo.Entry{id: "foo", __meta__: %{source: {nil, "test"}}}
@@ -49,7 +50,7 @@ defmodule Koype.Storage.JsonTest do
49 50
       entry = insert(:entry)
50 51
       reason = :fake_system_error
51 52
 
52
-      with_mock(Koype.Http, [], get: fn _ -> {:error, %Koype.Http.Error{reason: reason}} end) do
53
+      with_mock(Koype.Http, [], get: fn _ -> {:error, %Koype.Http.Error{reason: reason, raw: nil}} end) do
53 54
         assert {:error, type: :system_error, reason: ^reason} = Json.find(entry)
54 55
       end
55 56
     end
@@ -58,7 +59,7 @@ defmodule Koype.Storage.JsonTest do
58 59
       entry = insert(:entry)
59 60
       code = 404
60 61
 
61
-      with_mock(Koype.Http, [], get: fn _ -> {:ok, %Koype.Http.Response{code: code}} end) do
62
+      use_cassette :stub, url: "~r/*/", status_code: 404 do
62 63
         assert {:error, type: :network_error, code: ^code} = Json.find(entry)
63 64
       end
64 65
     end
@@ -68,11 +69,7 @@ defmodule Koype.Storage.JsonTest do
68 69
       data = Faker.Lorem.sentences()
69 70
       body = Jason.encode!(data)
70 71
 
71
-      with_mock(
72
-        Koype.Http,
73
-        [],
74
-        get: fn _ -> {:ok, %Koype.Http.Response{code: 200, body: body}} end
75
-      ) do
72
+      use_cassette :stub, url: "~r/*/", status_code: 200 do
76 73
         assert {:ok, ^data} = Json.find(entry)
77 74
       end
78 75
     end

+ 2
- 2
test/unit/webmention_test.exs View File

@@ -2,7 +2,7 @@ defmodule Koype.WebmentionTest do
2 2
   alias Koype.Webmention, as: Subject
3 3
 
4 4
   use Koype.DataCase
5
-  use ExVCR.Mock
5
+  use Koype.HttpMock
6 6
 
7 7
   import Mock
8 8
   import Koype.Factory
@@ -31,7 +31,7 @@ defmodule Koype.WebmentionTest do
31 31
         assert webmention.source == source_uri
32 32
         assert webmention.target == URI.parse(target_uri).path
33 33
         assert webmention.type == "like"
34
-        assert webmention.author == source_uri
34
+        assert webmention.author == source_uri <> "/"
35 35
       end
36 36
     end
37 37
 

+ 1
- 1
web/controllers/media_controller.ex View File

@@ -49,7 +49,7 @@ defmodule Koype.Web.MediaController do
49 49
       content_type = MIME.from_path(filename)
50 50
 
51 51
       conn
52
-      |> merge_resp_headers(Map.take(resp.headers, ~w(Accept-Ranges Content-Length Last-Modified Etag)))
52
+      |> merge_resp_headers(Keyword.take(resp.headers, ~w(Accept-Ranges Content-Length Last-Modified Etag)))
53 53
       |> put_layout(false)
54 54
       |> put_resp_header("content-type", content_type)
55 55
       |> send_resp(200, resp.body)

Loading…
Cancel
Save