Browse Source

feat(webmention): Add upgrader for JSON.

jackyalcine 7 months ago
parent
commit
c71b535d1d
Signed by: Jacky Alciné <yo@jacky.wtf> GPG Key ID: 537A4F904B15268D

+ 3
- 1
config/config.exs View File

@@ -36,6 +36,9 @@ config :koype, Koype.Repo,
36 36
   json_library: Jason,
37 37
   migration_primary_key: [id: :id, type: :uuid],
38 38
   migration_timestamsp: [type: :utc_datetime],
39
+  pool_timeout: 60_000,
40
+  pool_size: 128,
41
+  log: true,
39 42
   database: "priv/repo/db/#{Mix.env()}.db"
40 43
 
41 44
 config :logger, :console,
@@ -67,7 +70,6 @@ config :ex_aws, :hackney_opts,
67 70
 
68 71
 config :ex_aws,
69 72
   storage_dir: "koype",
70
-  debug_requests: true,
71 73
   access_key_id: {:system, :string, "OBJECT_STORAGE_ACCESS_KEY"},
72 74
   secret_access_key: {:system, :string, "OBJECT_STORAGE_SECRET_KEY"},
73 75
   region: "local"

+ 1
- 1
config/dev.exs View File

@@ -22,7 +22,7 @@ config :koype, Koype.Web.Endpoint,
22 22
     ]
23 23
   ]
24 24
 
25
-config :logger, level: :info
25
+config :logger, level: :debug
26 26
 
27 27
 config :phoenix, :stacktrace_depth, 50
28 28
 

+ 1
- 1
lib/http.ex View File

@@ -30,7 +30,7 @@ defmodule Koype.Http do
30 30
   @fake_human_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133"
31 31
   @user_agent "Koype/#{Koype.version()} (https://koype.net/faq/user-agent)"
32 32
 
33
-  def timeout, do: 30_000
33
+  def timeout, do: 10_000
34 34
 
35 35
   defmodule Response do
36 36
     @moduledoc "Represents a HTTP response."

+ 2
- 2
lib/indieweb/micropub/query.ex View File

@@ -61,6 +61,8 @@ defmodule IndieWeb.Micropub.Query do
61 61
     end
62 62
   end
63 63
 
64
+  def invoke(_, _), do: {:error, :invalid_action}
65
+
64 66
   defp do_fetch_source_of_entry(entry, queried_properties)
65 67
 
66 68
   defp do_fetch_source_of_entry(model_url, queried_properties) when is_binary(model_url) do
@@ -95,8 +97,6 @@ defmodule IndieWeb.Micropub.Query do
95 97
     end
96 98
   end
97 99
 
98
-  def invoke(_, _), do: {:error, :invalid_action}
99
-
100 100
   defp do_normalize_for_mf2(mf2_json, model) do
101 101
     Enum.reduce(mf2_json, %{}, fn
102 102
       {"content", %{"html" => html}}, acc -> Map.put(acc, "content", %{"html" => html})

+ 4
- 1
lib/repo/webmention.ex View File

@@ -59,7 +59,9 @@ defmodule Koype.Repo.Webmention do
59 59
       data = %{
60 60
         "accessed" => accessed_dt,
61 61
         "author" => author,
62
-        "source" => do_extract_source_mf2(mf2)
62
+        "source" => do_extract_source_mf2(mf2),
63
+        "mf2" => mf2,
64
+        "url" => mf2["url"] || record.source
63 65
       }
64 66
 
65 67
       Koype.Storage.Json.persist(record, data)
@@ -68,6 +70,7 @@ defmodule Koype.Repo.Webmention do
68 70
     defp do_extract_source_mf2(mf2) do
69 71
       %{
70 72
         "published" => mf2 |> IndieWeb.MF2.get_value!("published", [nil]) |> List.first(),
73
+        "updated" => mf2 |> IndieWeb.MF2.get_value!("updated", [nil]) |> List.first(),
71 74
         "title" => mf2 |> IndieWeb.MF2.get_value!("name", [nil]) |> List.first(),
72 75
         "content" => mf2 |> IndieWeb.MF2.get_value!("content", [nil]) |> List.first()
73 76
       }

+ 8
- 3
lib/webmention.ex View File

@@ -41,6 +41,8 @@ defmodule Koype.Webmention do
41 41
 
42 42
     @impl true
43 43
     def from_source_url(target_url) when not is_nil(target_url) do
44
+      Logger.debug("Attempting to determine raw value.", target_url: target_url)
45
+
44 46
       Enum.reduce_while(
45 47
         [
46 48
           &do_resolve_homepage_for_target/1,
@@ -93,7 +95,7 @@ defmodule Koype.Webmention do
93 95
         [Koype.Repo.Entry, Koype.Repo.Category],
94 96
         {:error, :no_mentionable_models},
95 97
         fn module, acc ->
96
-          Logger.info("Using #{module} to look up model from the URI #{target_url}.")
98
+          Logger.debug("Looking up model from URL.", module: module, uri: target_url)
97 99
 
98 100
           case module.resolve_from_uri(target_url) do
99 101
             {:error, _} -> {:cont, acc}
@@ -132,7 +134,6 @@ defmodule Koype.Webmention do
132 134
           end)
133 135
 
134 136
         :timer.apply_after(retry_after, __MODULE__, :send!, args)
135
-        Logger.info("Webmention has been ratelimited; resending in #{retry_after} milliseconds.")
136 137
 
137 138
       {:ok, resp} ->
138 139
         Logger.info("Webmention for #{inspect(source)} to #{target} sent!")
@@ -188,7 +189,11 @@ defmodule Koype.Webmention do
188 189
       |> Koype.Repo.all(source: args[:source], target: args[:target_url])
189 190
       |> Enum.map(&Koype.Repo.delete(&1))
190 191
 
191
-    Logger.warn("Dropped all relating Webmentions from the database.", args)
192
+    Logger.warn("Dropped all relating Webmentions from the database.",
193
+      source: args[:source],
194
+      target: args[:target_url]
195
+    )
196
+
192 197
     {:ok, :gone}
193 198
   end
194 199
 

+ 75
- 0
priv/repo/migrations/20190412031823_json_webmention_add_url_and_mf2.exs View File

@@ -0,0 +1,75 @@
1
+defmodule Koype.Repo.Migrations.JsonWebmentionAddUrlAndMf2 do
2
+  use Ecto.Migration
3
+  @disable_ddl_transaction true
4
+  require Logger
5
+
6
+  defp do_setup_app do
7
+    Enum.each(~w(ex_aws ex_aws_s3 arc)a, fn app ->
8
+      {:ok, _} = Application.ensure_all_started(app)
9
+    end)
10
+
11
+    Koype.Application.load_runtime_config()
12
+  end
13
+
14
+  def up do
15
+    do_setup_app()
16
+
17
+    Koype.Repo.Webmention
18
+    |> Koype.Repo.all()
19
+    |> Task.async_stream(
20
+      fn webmention ->
21
+        case Koype.Webmention.receive!(source: webmention.source, target: webmention.target) do
22
+          {:ok, _} ->
23
+            Logger.info("Updated the Webmention with the new JSON layout.", id: webmention.id)
24
+            :ok
25
+
26
+          error ->
27
+            Logger.warn("Failed to reparse and upgrade the JSON for the webmention.",
28
+              error: inspect(error),
29
+              source: webmention.source,
30
+              target: webmention.target,
31
+              id: webmention.id
32
+            )
33
+
34
+            raise error
35
+        end
36
+      end,
37
+      ordered: false,
38
+      max_concurrency: 100,
39
+      timeout: 60_000,
40
+      on_timeout: :kill_task
41
+    )
42
+    |> Stream.each(fn v -> v == :ok end)
43
+    |> Stream.run()
44
+  end
45
+
46
+  def down do
47
+    do_setup_app()
48
+
49
+    Koype.Repo.Webmention
50
+    |> Koype.Repo.all()
51
+    |> Task.async_stream(
52
+      fn webmention ->
53
+        with(
54
+          {:ok, json} <- Koype.Repo.Webmention.Json.find(webmention),
55
+          restored_json <- Map.drop(json, ~w(mf2 url)),
56
+          {:ok, _} <- Koype.Storage.Json.persist(webmention, restored_json)
57
+        ) do
58
+          :ok
59
+        else
60
+          error ->
61
+            Logger.error("Failed to restore JSON for Webmention.",
62
+              id: webmention.id,
63
+              error: inspect(error)
64
+            )
65
+        end
66
+      end,
67
+      ordered: false,
68
+      max_concurrency: 100,
69
+      timeout: 60_000,
70
+      on_timeout: :kill_task
71
+    )
72
+    |> Stream.each(fn v -> v == :ok end)
73
+    |> Stream.run()
74
+  end
75
+end

+ 1
- 7
web/controllers/auth_controller.ex View File

@@ -29,13 +29,7 @@ defmodule Koype.Web.AuthController do
29 29
   defp do_redirect(conn, params)
30 30
 
31 31
   defp do_redirect(conn, %{"redirect_to" => uri} = _params) do
32
-    decoded_uri = Base.url_decode64!(uri)
33
-
34
-    if String.starts_with?(decoded_uri, "/") do
35
-      redirect(conn, to: decoded_uri)
36
-    else
37
-      redirect(conn, external: decoded_uri)
38
-    end
32
+    redirect(conn, external: uri)
39 33
   end
40 34
 
41 35
   defp do_redirect(conn, _params) do

+ 1
- 1
web/endpoint.ex View File

@@ -42,7 +42,7 @@ defmodule Koype.Web.Endpoint do
42 42
     Plug.Parsers,
43 43
     parsers: [:urlencoded, :multipart, :json],
44 44
     pass: ["*/*"],
45
-    json_decoder: Poison
45
+    json_decoder: Phoenix.json_library()
46 46
   )
47 47
 
48 48
   plug(Plug.MethodOverride)

+ 1
- 5
web/plug/auth_handler/browser.ex View File

@@ -21,10 +21,6 @@ defmodule Koype.Web.Plug.AuthBrowserErrorHandler do
21 21
   import Koype.Web.Router.Helpers, only: [auth_path: 3]
22 22
 
23 23
   def auth_error(conn, {_type, _reason}, _opts) do
24
-    uri = Base.url_encode64(current_path(conn))
25
-    path = auth_path(conn, :login, redirect_to: uri)
26
-
27
-    conn
28
-    |> redirect(to: path)
24
+    redirect(conn, to: auth_path(conn, :login, redirect_to: current_path(conn)))
29 25
   end
30 26
 end

+ 4
- 3
web/templates/layout/_meta.html.eex View File

@@ -1,13 +1,14 @@
1
-<meta charset="utf-8">
2 1
 <title><%= Koype.Web.LayoutView.title(assigns) %></title>
2
+<meta charset="utf-8">
3 3
 <meta name="viewport" content="initial-scale=1.0,user-scalable=yes,width=device-width" />
4 4
 <%= Koype.Web.LayoutView.header_tags(:meta, assigns) %>
5 5
 <%= Koype.Web.LayoutView.header_tags(:link, assigns) %><%= if Map.get(assigns, "theme_id", "default") == "default" do %><link rel="icon" href="<%= Koype.Profile.photo(:square) || asset_path_for(:core, "images/logo/icon.svg") %>" />
6
+<link rel="preload" href="<%= asset_path_for(:core, "fonts.css") %>" as="style" />
6 7
 <link rel="preload" href="<%= asset_path_for(:core, "koype.css") %>" as="style" />
7
-<link rel="stylesheet" href="<%= asset_path_for(:core, "vendor.css") %>" />
8
+<link rel="preload" href="<%= asset_path_for(:core, "vendor.css") %>" as="style" />
8 9
 <link rel="stylesheet" href="<%= asset_path_for(:core, "koype.css") %>" />
10
+<link rel="stylesheet" href="<%= asset_path_for(:core, "vendor.css") %>" />
9 11
 <noscript>
10 12
 <link rel="stylesheet" href="<%= asset_path_for(:core, "fonts.css") %>" />
11
-<link rel="preload" href="<%= asset_path_for(:core, "fonts.css") %>" as="style" />
12 13
 </noscript>
13 14
 <% end %>

+ 3
- 3
web/views/entry_view.ex View File

@@ -51,14 +51,14 @@ defmodule Koype.Web.EntryView do
51 51
   end
52 52
 
53 53
   def tag({:meta, :itemprop}, "entry/view", assigns) do
54
+    photo_uri = Map.get(assigns["entry"]["json"], "photo", [Koype.Profile.photo()])
55
+
54 56
     [
55 57
       %{itemprop: :description, content: assigns["entry"]["json"]["content"]["plain"] |> truncate},
56 58
       %{itemprop: :name, content: assigns["entry"]["title"]},
57 59
       %{
58 60
         itemprop: :image,
59
-        content:
60
-          assigns["entry"]["json"]["photo"] |> List.first() |> Map.get("url") ||
61
-            Koype.Profile.photo()
61
+        content: photo_uri
62 62
       }
63 63
     ]
64 64
   end

Loading…
Cancel
Save