An IndieWeb engine for a self-hostable website. https://koype.net/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

webmention.ex 9.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. # Koype: a IndieWeb-focused, single-tenant website engine for people.
  2. #
  3. # Copyright © 2019 Jacky Alciné <jacky.is@black.af>
  4. #
  5. # This file belongs to the Koype project.
  6. #
  7. # This program is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU Affero General Public License as published by
  9. # the Free Software Foundation, either version 3 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU Affero General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU Affero General Public License
  18. # along with this program. If not, see <https://www.gnu.org/licenses/>.
  19. defmodule Koype.Webmention do
  20. @moduledoc false
  21. require Koype.Http
  22. require Ecto.Query
  23. require Logger
  24. defmodule URIAdapter do
  25. @moduledoc false
  26. @behaviour IndieWeb.Webmention.URIAdapter
  27. @impl true
  28. def to_source_url(target_url)
  29. def to_source_url(target_url) when is_binary(target_url), do: URI.parse(target_url)
  30. def to_source_url(%Koype.Repo.Entry{} = entry),
  31. do: to_source_url(Koype.Repo.Entry.get_uri(entry))
  32. def to_source_url(%URI{} = target_url), do: target_url
  33. def to_source_url(_), do: nil
  34. @impl true
  35. def from_source_url(%URI{} = target_url), do: from_source_url(URI.to_string(target_url))
  36. @impl true
  37. def from_source_url(target_url) when not is_nil(target_url) do
  38. Logger.debug("Attempting to determine raw value.", target_url: target_url)
  39. Enum.reduce_while(
  40. [
  41. &do_resolve_homepage_for_target/1,
  42. &do_resolve_model_for_target/1,
  43. &do_resolve_if_on_site/1
  44. ],
  45. nil,
  46. fn approach, acc ->
  47. case approach.(target_url) do
  48. {:error, _} -> {:cont, acc}
  49. {:ok, result} -> {:halt, result}
  50. end
  51. end
  52. )
  53. end
  54. def from_source_url(nil), do: from_source_url("/")
  55. defp do_resolve_if_on_site(target_url) when is_binary(target_url) do
  56. current_host = URI.parse(Koype.host()).host
  57. target_host = URI.parse(target_url).host
  58. if target_host == current_host || String.starts_with?(target_url, "/") do
  59. {:ok, target_url}
  60. else
  61. {:error, :not_on_site}
  62. end
  63. end
  64. defp do_resolve_homepage_for_target(target_url) when is_binary(target_url) do
  65. current_host = Koype.host()
  66. cond do
  67. "/" == target_url || current_host == target_url ->
  68. Logger.info("#{target_url} matches our homepage.")
  69. {:ok, :homepage}
  70. URI.parse(current_host) == URI.parse(target_url |> String.trim_trailing("/")) ->
  71. Logger.info("#{target_url} matches our homepage directly.")
  72. {:ok, :homepage}
  73. true ->
  74. Logger.debug(fn -> "The URI #{target_url} is not the homepage (#{current_host})." end)
  75. {:error, :not_homepage}
  76. end
  77. end
  78. defp do_resolve_model_for_target(target_url) when is_binary(target_url) do
  79. Enum.reduce_while(
  80. [Koype.Repo.Entry, Koype.Repo.Category],
  81. {:error, :no_mentionable_models},
  82. fn module, acc ->
  83. Logger.debug("Looking up model from URL.", module: module, uri: target_url)
  84. case module.resolve_from_uri(target_url) do
  85. {:error, _} -> {:cont, acc}
  86. {:ok, model} -> {:halt, {:ok, model}}
  87. end
  88. end
  89. )
  90. end
  91. end
  92. def endpoint(), do: Koype.Web.Router.Helpers.indie_webmention_url(Koype.Web.Endpoint, :incoming)
  93. def send(args), do: Koype.Job.create(Koype.Job.Webmention.Send, args)
  94. def send!(args) do
  95. source = args[:source]
  96. target = args[:target]
  97. case IndieWeb.Webmention.send(target, source) do
  98. {:ok, %Koype.Http.Response{code: code, headers: headers}} when code == 429 ->
  99. retry_after =
  100. headers
  101. |> Enum.find_value(fn
  102. {"Retry-After", count} ->
  103. with(
  104. :error <- Integer.parse(count),
  105. {:ok, retry_dt} <- Calendar.DateTime.Parse.httpdate!(count)
  106. ) do
  107. DateTime.diff(retry_dt, DateTime.utc_now(), :millisecond)
  108. else
  109. seconds when is_integer(seconds) -> :timer.seconds(seconds)
  110. end
  111. _ ->
  112. false
  113. end)
  114. :timer.apply_after(retry_after, __MODULE__, :send!, args)
  115. {:ok, resp} ->
  116. Logger.info("Webmention for #{inspect(source)} to #{target} sent!")
  117. {:ok, resp}
  118. error ->
  119. Logger.error(
  120. "Failed to send Webmention to #{target} from #{inspect(source)}: #{inspect(error)}"
  121. )
  122. error
  123. end
  124. end
  125. def receive(args), do: Koype.Job.create(Koype.Job.Webmention.Receive, args)
  126. def receive!(args) do
  127. case IndieWeb.Webmention.receive(args) do
  128. {:ok, args} -> do_build_webmention(args)
  129. error -> error
  130. end
  131. end
  132. defp do_build_webmention(args) do
  133. target_path =
  134. case args[:target_url] |> URI.parse() |> Map.get(:path) do
  135. "" -> "/"
  136. path -> path
  137. end
  138. args = Keyword.put(args, :target_url, target_path)
  139. case Koype.Http.get(args[:source]) do
  140. {:ok, %Koype.Http.Response{code: code}} when code in [404, 410] ->
  141. do_deletion_of_webmention(args)
  142. {:ok, %Koype.Http.Response{code: code}} when code >= 200 and code < 300 ->
  143. do_upsertion_of_webmention(args)
  144. {:ok, %Koype.Http.Response{code: code} = resp} when code >= 400 and code < 599 ->
  145. Logger.info("The remote page is not usable for MF2 fetching.", code: code)
  146. {:error, reason: :page_unavailable, raw: resp}
  147. {:error, error} ->
  148. Logger.info("The remote page is not usable for MF2 fetching.", error: error)
  149. {:error, reason: :page_down, raw: error}
  150. end
  151. end
  152. defp do_deletion_of_webmention(args) do
  153. mentions =
  154. Koype.Repo.Webmention
  155. |> Ecto.Query.where(source: ^args[:source], target: ^args[:target_url])
  156. |> Koype.Repo.all()
  157. |> Enum.map(&Koype.Repo.Webmention.delete(&1))
  158. Logger.warn("Marked all relating Webmentions as deleted.",
  159. source: args[:source],
  160. target: args[:target_url],
  161. deleted_count: Enum.count(mentions)
  162. )
  163. {:ok, :gone}
  164. end
  165. defp do_upsertion_of_webmention(args) do
  166. target_url = args[:target_url]
  167. source = args[:source]
  168. id =
  169. case Koype.Repo.get_by(Koype.Repo.Webmention, source: source) do
  170. nil -> nil
  171. %{id: id} -> id
  172. end
  173. with(
  174. {:ok, mf2} when is_map(mf2) <- IndieWeb.MF2.Remote.fetch(source),
  175. entry_mf2 when is_map(entry_mf2) <- IndieWeb.MF2.get_format(mf2, "entry"),
  176. {:ok, author} <- IndieWeb.HCard.resolve(source)
  177. ) do
  178. params = %{
  179. id: id,
  180. source: source,
  181. target: target_url,
  182. type: args[:type] || "note",
  183. mf2: entry_mf2
  184. }
  185. Logger.info("Saving Webmention from #{source} to #{target_url}...")
  186. case Koype.Repo.upsert(Koype.Repo.Webmention, params) do
  187. {:ok, record} ->
  188. # NOTE: Should we make these actions async.
  189. final_record =
  190. record
  191. |> Koype.Repo.Webmention.update_author!(author)
  192. |> Koype.Repo.Webmention.update_type!()
  193. |> Koype.Repo.Webmention.attempt_auto_moderation()
  194. {:ok, final_record}
  195. {:error, error} = err ->
  196. Logger.error(
  197. "Failed to create Webmention from #{source} to #{target_url}: #{inspect(error)}."
  198. )
  199. IndieWeb.MF2.Remote.flush(source)
  200. err
  201. end
  202. else
  203. nil ->
  204. Logger.warn("Failed to fetch MF2 of #{source}; no entry .")
  205. IndieWeb.MF2.Remote.flush(inspect(source))
  206. {:error, :mf2_not_found}
  207. {:ok, nil} ->
  208. Logger.warn("No MF2 found for #{source}.")
  209. {:error, :mf2_not_found}
  210. {:error, error} = err ->
  211. Logger.warn("Failed to fetch MF2 of #{source}: #{inspect(error)}.")
  212. IndieWeb.MF2.Remote.flush(inspect(source))
  213. err
  214. end
  215. end
  216. def fetch_for(entry, type \\ :all)
  217. def fetch_for(%Koype.Repo.Entry{} = entry, type) do
  218. entry
  219. |> Koype.Repo.Entry.get_uri()
  220. |> fetch_for(type)
  221. end
  222. def fetch_for(path, type) do
  223. path
  224. |> Koype.Repo.Webmention.all(type)
  225. |> Enum.map(fn model ->
  226. with({:ok, json} <- Koype.Repo.Webmention.json_find(model)) do
  227. json
  228. |> Map.put("id", model.id)
  229. |> Map.put("url", model.source)
  230. |> Map.put("type", model.type)
  231. else
  232. {:error, _} -> nil
  233. end
  234. end)
  235. |> Enum.filter(&is_map/1)
  236. end
  237. def reprocess(), do: Koype.Job.create(Koype.Job.Webmention.Process, [])
  238. def default_moderation_status(),
  239. do: Koype.Setting.get("webmention:moderation_status", "pending") |> String.to_existing_atom()
  240. def set_default_moderation_status(status \\ "pending")
  241. def set_default_moderation_status(status) when status in ~w(pending approved rejected),
  242. do: Koype.Setting.set("webmention:moderation_status", status)
  243. def set_default_moderation_status(status) when is_binary(status),
  244. do: set_default_moderation_status(Atom.to_string(status))
  245. def set_default_moderation_status(_), do: set_default_moderation_status("pending")
  246. end