Browse Source

fix(entry): Update logic around webmentions + titles.

jackyalcine 1 month ago
parent
commit
f05367191c
Signed by: Jacky Alciné <yo@jacky.wtf> GPG Key ID: 537A4F904B15268D

+ 4
- 1
lib/post.ex View File

@@ -144,7 +144,7 @@ defmodule Koype.Post do
144 144
         when is_map(response_props) do
145 145
       url = Map.get(response_props, "url", []) |> List.wrap() |> List.first()
146 146
 
147
-      title =
147
+      computed_title =
148 148
         cond do
149 149
           Map.has_key?(response_props, "name") ->
150 150
             name = Map.get(response_props, "name") |> List.wrap() |> List.first()
@@ -168,6 +168,9 @@ defmodule Koype.Post do
168 168
             |> URI.parse()
169 169
             |> Map.get(:host)
170 170
         end
171
+
172
+      title =
173
+        computed_title
171 174
         |> String.trim()
172 175
 
173 176
       Enum.join([unquote(prefix), title], " ")

+ 1
- 6
lib/repo.ex View File

@@ -118,13 +118,8 @@ defmodule Koype.Repo do
118 118
     end
119 119
   end
120 120
 
121
-  def upsert(model, attrs) when is_map(attrs) do
122
-    params = attrs |> Map.to_list() |> Keyword.new()
123
-    upsert(model, params)
124
-  end
125
-
126 121
   def upsert(model, attrs) do
127
-    if is_nil(Keyword.get(attrs, :id, nil)) do
122
+    if is_nil(attrs[:id]) do
128 123
       model.create(attrs)
129 124
     else
130 125
       model.update(Koype.Repo.get(model, attrs[:id]), attrs)

+ 6
- 0
lib/repo/entry.ex View File

@@ -213,6 +213,12 @@ defmodule Koype.Repo.Entry do
213 213
     is_built_name_empty = is_nil(model.name) || model.name == "Entry"
214 214
     plain_content = properties["content"]["plain"] || properties["content"]["text"]
215 215
 
216
+    Logger.info("Attempting to generate a name for this model.",
217
+      entry_id: model.id,
218
+      type: model.type,
219
+      properties: inspect(properties)
220
+    )
221
+
216 222
     name =
217 223
       cond do
218 224
         !is_built_name_empty && is_provided_name_empty ->

+ 54
- 36
lib/repo/webmention.ex View File

@@ -24,16 +24,16 @@ defmodule Koype.Repo.Webmention do
24 24
   This is the structure that the JSON data is stored as.
25 25
   ```elixir
26 26
   %{
27
-    "source" => %{
28
-      "published" => Calendar.DateTime,
29
-      "updated" => Calendar.DateTime,
30
-      "title" => String,
31
-      "content" => Map
32
-    },
33
-    "accessed" => Calendar.DateTime,
34
-    "author" => Map, // check IndieWeb.HCard
35
-    "mf2" => Map,
36
-    "url" => String
27
+  "source" => %{
28
+  "published" => Calendar.DateTime,
29
+  "updated" => Calendar.DateTime,
30
+  "title" => String,
31
+  "content" => Map
32
+  },
33
+  "accessed" => Calendar.DateTime,
34
+  "author" => Map, // check IndieWeb.HCard
35
+  "mf2" => Map,
36
+  "url" => String
37 37
   }
38 38
   ```
39 39
   """
@@ -68,6 +68,10 @@ defmodule Koype.Repo.Webmention do
68 68
   @doc false
69 69
   def transform_data(data: data, record: record) do
70 70
     mf2 = Map.get(data, :mf2, %{})
71
+
72
+    accessed_dt =
73
+      Calendar.DateTime.Format.rfc3339(Enum.max([record.updated_at, record.inserted_at]), 3)
74
+
71 75
     author = do_extract_author(record, data)
72 76
 
73 77
     if mf2 == %{} || is_nil(mf2) do
@@ -77,24 +81,33 @@ defmodule Koype.Repo.Webmention do
77 81
       )
78 82
     end
79 83
 
80
-    accessed_dt =
81
-      Calendar.DateTime.Format.rfc3339(Enum.max([record.updated_at, record.inserted_at]), 3)
82
-
83
-    source_mf2 = %{
84
-      "published" => mf2 |> IndieWeb.MF2.get_value!("published", [""]) |> List.first(),
85
-      "updated" => mf2 |> IndieWeb.MF2.get_value!("updated", [""]) |> List.first(),
86
-      "title" => mf2 |> IndieWeb.MF2.get_value!("name", [""]) |> List.first(),
87
-      "content" =>
88
-        mf2 |> IndieWeb.MF2.get_value!("content", [""]) |> Enum.filter(&is_map/1) |> List.first()
89
-    }
90
-
91
-    %{
92
-      "accessed" => accessed_dt,
93
-      "author" => author,
94
-      "source" => source_mf2,
95
-      "mf2" => mf2,
96
-      "url" => mf2["url"] || record.source
97
-    }
84
+    %{"accessed" => accessed_dt, "author" => author}
85
+    |> Map.put_new_lazy("mf2", fn ->
86
+      unless is_nil(mf2) do
87
+        mf2
88
+      end
89
+    end)
90
+    |> Map.put_new_lazy("source", fn ->
91
+      unless is_nil(mf2) do
92
+        %{
93
+          "published" => mf2 |> IndieWeb.MF2.get_value!("published", [""]) |> List.first(),
94
+          "updated" => mf2 |> IndieWeb.MF2.get_value!("updated", [""]) |> List.first(),
95
+          "title" => mf2 |> IndieWeb.MF2.get_value!("name", [""]) |> List.first(),
96
+          "content" =>
97
+            mf2
98
+            |> IndieWeb.MF2.get_value!("content", [""])
99
+            |> Enum.filter(&is_map/1)
100
+            |> List.first()
101
+        }
102
+      end
103
+    end)
104
+    |> Map.put_new_lazy("url", fn ->
105
+      if is_nil(mf2) do
106
+        record.source
107
+      else
108
+        mf2 |> IndieWeb.MF2.get_value!("url", [record.source]) |> List.first()
109
+      end
110
+    end)
98 111
   end
99 112
 
100 113
   defp do_extract_author(record, args)
@@ -137,6 +150,8 @@ defmodule Koype.Repo.Webmention do
137 150
 
138 151
   @doc false
139 152
   def changeset(webmention, attrs) do
153
+    excluded_source_values = [attrs[:target]]
154
+
140 155
     webmention
141 156
     |> cast(attrs, @required_attrs ++ @optional_attrs)
142 157
     |> validate_required(@required_attrs)
@@ -144,7 +159,7 @@ defmodule Koype.Repo.Webmention do
144 159
     |> unique_constraint(:source, name: :webmentions_unique_mention_index)
145 160
     |> unique_constraint(:target, name: :webmentions_unique_mention_index)
146 161
     |> unique_constraint(:type, name: :webmentions_unique_mention_index)
147
-    |> validate_exclusion(:source, [attrs[:target]])
162
+    |> validate_exclusion(:source, excluded_source_values)
148 163
   end
149 164
 
150 165
   def update_author(model, author \\ nil)
@@ -176,7 +191,7 @@ defmodule Koype.Repo.Webmention do
176 191
     ) do
177 192
       types = IndieWeb.Post.extract_types(entry_mf2["properties"])
178 193
       type = IndieWeb.Post.determine_type(entry_mf2["properties"], types) |> Atom.to_string()
179
-      update(webmention, type: type, mf2: entry_mf2) |> elem(1)
194
+      update(webmention, %{type: type, mf2: entry_mf2}) |> elem(1)
180 195
     else
181 196
       {:error, _} ->
182 197
         webmention
@@ -220,9 +235,10 @@ defmodule Koype.Repo.Webmention do
220 235
 
221 236
   def update(model, args) do
222 237
     with(
223
-      cs <- model |> changeset(%{}) |> change(args |> Keyword.drop(~w(mf2)a)),
238
+      cs <- changeset(model, args),
224 239
       {:ok, record} <- Koype.Repo.update(cs),
225
-      {:ok, _} <- json_update(record, %{mf2: args[:mf2], author: args[:author] || record.author})
240
+      {:ok, _} <-
241
+        json_update(record, %{mf2: args[:mf2], author: Map.get(args, :author, record.author)})
226 242
     ) do
227 243
       Logger.info("Updated Webmention.",
228 244
         author: record.author,
@@ -245,7 +261,9 @@ defmodule Koype.Repo.Webmention do
245 261
   def query(uri, type) when is_binary(uri), do: query(URI.parse(uri), type)
246 262
   def query(%URI{path: path}, :all), do: from(w in Koype.Repo.Webmention) |> where(target: ^path)
247 263
   def query(%URI{} = uri, type) when is_binary(type), do: uri |> query(:all) |> where(type: ^type)
248
-  def query(%URI{} = uri, type) when is_atom(type), do: query(uri, Atom.to_string(type))
264
+
265
+  def query(%URI{} = uri, type) when is_atom(type),
266
+    do: uri |> query(:all) |> where(type: ^Atom.to_string(type))
249 267
 
250 268
   @doc "Fetches the Webmentions with the specified target and optionally of provided type."
251 269
   @spec all(path :: URI.t() | binary(), type :: atom()) :: list(Webmention.t()) | nil
@@ -259,9 +277,9 @@ defmodule Koype.Repo.Webmention do
259 277
   def count_of(path, type), do: path |> query(type) |> Koype.Repo.count()
260 278
 
261 279
   def set_moderation_status(webmention, status \\ :pending) do
262
-    update(webmention,
263
-      moderation_status: Atom.to_string(status),
264
-      moderated_at: Calendar.DateTime.now_utc()
280
+    update(
281
+      webmention,
282
+      %{moderation_status: Atom.to_string(status), moderated_at: Calendar.DateTime.now_utc()}
265 283
     )
266 284
   end
267 285
 

+ 4
- 3
lib/webmention.ex View File

@@ -215,18 +215,19 @@ defmodule Koype.Webmention do
215 215
       entry_mf2 when is_map(entry_mf2) <- IndieWeb.MF2.get_format(mf2, "entry"),
216 216
       {:ok, author} <- IndieWeb.HCard.resolve(source)
217 217
     ) do
218
-      params = [
218
+      params = %{
219 219
         id: id,
220 220
         source: source,
221 221
         target: target_url,
222
-        type: Keyword.get(args, :type, "note"),
222
+        type: args[:type] || "note",
223 223
         mf2: entry_mf2
224
-      ]
224
+      }
225 225
 
226 226
       Logger.info("Saving Webmention from #{source} to #{target_url}...")
227 227
 
228 228
       case Koype.Repo.upsert(Koype.Repo.Webmention, params) do
229 229
         {:ok, record} ->
230
+          # NOTE: Should we make these actions async.
230 231
           final_record =
231 232
             record
232 233
             |> Koype.Repo.Webmention.update_author!(author)

+ 7
- 7
test/test_helper.exs View File

@@ -3,13 +3,13 @@
3 3
   {:ok, _pid} = Application.ensure_all_started(app)
4 4
 end)
5 5
 
6
-Bureaucrat.start(
7
-  env_var: "DOC",
8
-  writer: Bureaucrat.MarkdownWriter,
9
-  default_path: "docs/api/source/index.html.md",
10
-  swagger: "priv/static/swagger.json" |> File.read!() |> Jason.decode!(),
11
-  json_library: Jason
12
-)
6
+# Bureaucrat.start(
7
+#   env_var: "DOC",
8
+#   writer: Bureaucrat.MarkdownWriter,
9
+#   default_path: "docs/api/source/index.html.md",
10
+#   swagger: "priv/static/swagger.json" |> File.read!() |> Jason.decode!(),
11
+#   json_library: Jason
12
+# )
13 13
 
14 14
 Ecto.Adapters.SQL.Sandbox.mode(Koype.Repo, :manual)
15 15
 

+ 25
- 0
test/unit/post_test.exs View File

@@ -151,6 +151,29 @@ defmodule Koype.PostTest do
151 151
         assert String.contains?(obtained_title, name)
152 152
       end
153 153
 
154
+      test "extract title from name in reply content of #{type}" do
155
+        name = Faker.Lorem.sentence()
156
+
157
+        base_entry_json =
158
+          build(:entry_json)
159
+          |> with_post_type(unquote(type))
160
+
161
+        reply_mf2 =
162
+          base_entry_json
163
+          |> Map.get(unquote(key))
164
+          |> List.wrap()
165
+          |> List.first()
166
+          |> Map.merge(%{"name" => [name]})
167
+          |> List.wrap()
168
+
169
+        entry_json =
170
+          Map.put(base_entry_json, unquote(key), reply_mf2) |> Map.drop(~w(name content)s)
171
+
172
+        obtained_title = Subject.determine_title(unquote(type), entry_json)
173
+        assert String.contains?(obtained_title, unquote(prefix))
174
+        assert String.contains?(obtained_title, name)
175
+      end
176
+
154 177
       test "extract title of #{type} via provided content" do
155 178
         entry_json =
156 179
           build(:entry_json)
@@ -181,6 +204,8 @@ defmodule Koype.PostTest do
181 204
               entry_json[unquote(key)]
182 205
               |> List.first()
183 206
               |> Map.get("url")
207
+              |> List.wrap()
208
+              |> List.first()
184 209
               |> URI.parse()
185 210
               |> Map.get(:host)
186 211
 

+ 1
- 1
test/unit/repo/webmention_test.exs View File

@@ -66,7 +66,7 @@ defmodule Koype.Repo.WebmentionTest do
66 66
 
67 67
       webmention = insert(:webmention, author: "https://jacky.wtf", source: "https://jacky.wtf")
68 68
       Koype.Repo.Webmention.json_persist(webmention, %{mf2: mf2, author: "https://jacky.wtf"})
69
-      assert {:ok, updated_wm} = Subject.update(webmention, mf2: new_mf2)
69
+      assert {:ok, updated_wm} = Subject.update(webmention, %{mf2: new_mf2})
70 70
       assert {:ok, updated_wm_json} = updated_wm |> Koype.Repo.Webmention.json_find()
71 71
       assert "Attending WaffleJS // January 2019" = updated_wm_json["source"]["title"]
72 72
     end

+ 6
- 3
test/unit/webmention_test.exs View File

@@ -9,6 +9,7 @@ defmodule Koype.WebmentionTest do
9 9
   import Koype.Factory
10 10
 
11 11
   @source_uri "https://raw.githubusercontent.com/microformats/tests/master/tests/microformats-v2/h-entry/summarycontent.html"
12
+  @empty_source %{"content" => nil, "published" => "", "title" => "", "updated" => ""}
12 13
 
13 14
   doctest Subject
14 15
 
@@ -25,6 +26,7 @@ defmodule Koype.WebmentionTest do
25 26
 
26 27
       assert wm_json = Model.json_find!(webmention)
27 28
       refute wm_json["mf2"] == %{}
29
+      refute wm_json["source"] == @empty_source
28 30
     end
29 31
 
30 32
     test "stores new for homepage mention using whole URI" do
@@ -35,6 +37,7 @@ defmodule Koype.WebmentionTest do
35 37
 
36 38
       assert wm_json = Model.json_find!(webmention)
37 39
       refute wm_json["mf2"] == %{}
40
+      refute wm_json["source"] == @empty_source
38 41
     end
39 42
 
40 43
     test "stores new for homepage mention using only path" do
@@ -45,6 +48,7 @@ defmodule Koype.WebmentionTest do
45 48
 
46 49
       assert wm_json = Model.json_find!(webmention)
47 50
       refute wm_json["mf2"] == %{}
51
+      refute wm_json["source"] == @empty_source
48 52
     end
49 53
 
50 54
     test "stores new for arbitrary page on site" do
@@ -57,6 +61,7 @@ defmodule Koype.WebmentionTest do
57 61
 
58 62
       assert wm_json = Model.json_find!(webmention)
59 63
       refute wm_json["mf2"] == %{}
64
+      refute wm_json["source"] == @empty_source
60 65
     end
61 66
 
62 67
     test "updates existing - set to 410 / deleted" do
@@ -96,6 +101,7 @@ defmodule Koype.WebmentionTest do
96 101
 
97 102
       assert wm_json = Model.json_find!(webmention)
98 103
       refute wm_json["mf2"] == %{}
104
+      refute wm_json["source"] == @empty_source
99 105
     end
100 106
 
101 107
     test "fails when MF2 of source fails" do
@@ -143,8 +149,5 @@ defmodule Koype.WebmentionTest do
143 149
       assert {:error, [reason: :page_unavailable, raw: %Koype.Http.Response{}]} =
144 150
                Subject.receive!(source: "https://httpbin.org/status/500", target: target_uri)
145 151
     end
146
-
147
-    @tag skip: true
148
-    test "ensures MF2 of remote site is imported to storage JSON"
149 152
   end
150 153
 end

Loading…
Cancel
Save