total rebaseHEADmaster

6 files changed, 186 insertions, 187 deletions

diff --git a/test/pleroma/web/rich_media/card_test.exs b/test/pleroma/web/rich_media/card_test.exs
new file mode 100644
index 0000000..516ac99
--- /dev/null
+++ b/test/pleroma/web/rich_media/card_test.exs
@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.CardTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.RichMedia.Card
+
+  import Mox
+  import Pleroma.Factory
+  import Tesla.Mock
+
+  setup do
+    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    :ok
+  end
+
+  setup do: clear_config([:rich_media, :enabled], true)
+
+  test "crawls URL in activity" do
+    user = insert(:user)
+
+    url = "https://example.com/ogp"
+    url_hash = Card.url_to_hash(url)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "[test](#{url})",
+        content_type: "text/markdown"
+      })
+
+    assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity)
+  end
+
+  test "recrawls URLs on status edits/updates" do
+    original_url = "https://google.com/"
+    original_url_hash = Card.url_to_hash(original_url)
+    updated_url = "https://yahoo.com/"
+    updated_url_hash = Card.url_to_hash(updated_url)
+
+    user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{status: "I like this site #{original_url}"})
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+
+    assert match?(
+             %Card{url_hash: ^original_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+
+    {:ok, _} = CommonAPI.update(user, activity, %{status: "I like this site #{updated_url}"})
+
+    activity = Pleroma.Activity.get_by_id(activity.id)
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+
+    assert match?(
+             %Card{url_hash: ^updated_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+  end
+end
diff --git a/test/pleroma/web/rich_media/helpers_test.exs b/test/pleroma/web/rich_media/helpers_test.exs
deleted file mode 100755
index 630b3ca..0000000
--- a/test/pleroma/web/rich_media/helpers_test.exs
+++ /dev/null
@@ -1,84 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.RichMedia.HelpersTest do
-  use Pleroma.DataCase
-
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.RichMedia.Helpers
-
-  import Pleroma.Factory
-  import Tesla.Mock
-
-  setup do
-    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-    :ok
-  end
-
-  setup do: clear_config([:rich_media, :enabled])
-
-  test "refuses to crawl incomplete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl malformed URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com[]/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "crawls valid, complete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](https://example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{page_url: "https://example.com/ogp", rich_media: _} =
-             Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl URLs of private network from posts" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{status: "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
-
-    {:ok, activity2} = CommonAPI.post(user, %{status: "https://10.111.10.1/notice/9kCP7V"})
-    {:ok, activity3} = CommonAPI.post(user, %{status: "https://172.16.32.40/notice/9kCP7V"})
-    {:ok, activity4} = CommonAPI.post(user, %{status: "https://192.168.10.40/notice/9kCP7V"})
-    {:ok, activity5} = CommonAPI.post(user, %{status: "https://pleroma.local/notice/9kCP7V"})
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} = Helpers.fetch_data_for_activity(activity)
-    assert %{} = Helpers.fetch_data_for_activity(activity2)
-    assert %{} = Helpers.fetch_data_for_activity(activity3)
-    assert %{} = Helpers.fetch_data_for_activity(activity4)
-    assert %{} = Helpers.fetch_data_for_activity(activity5)
-  end
-end
diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
index 59b3330..cd8be86 100755..100644
--- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
+++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
@@ -3,8 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
-  # Relies on Cachex, needs to be synchronous
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: false
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
 
   test "s3 signed url is parsed correct for expiration time" do
     url = "https://pleroma.social/amz"
@@ -22,7 +36,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
     expire_time =
       Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till)
 
-    assert {:ok, expire_time} == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
+    assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
   end
 
   test "s3 signed url is parsed and correct ttl is set for rich media" do
@@ -43,26 +57,29 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
     <meta name="twitter:site" content="Pleroma" />
     <meta name="twitter:title" content="Pleroma" />
     <meta name="twitter:description" content="Pleroma" />
-    <meta name="twitter:image" content="#{Map.get(metadata, :image)}" />
+    <meta name="twitter:image" content="#{Map.get(metadata, "image")}" />
     """
 
     Tesla.Mock.mock(fn
       %{
         method: :get,
-        url: "https://pleroma.social/amz"
+        url: ^url
       } ->
         %Tesla.Env{status: 200, body: body}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
     end)
 
-    Cachex.put(:rich_media_cache, url, metadata)
+    Card.get_or_backfill_by_url(url)
+
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
 
-    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url)
+    [%Oban.Job{scheduled_at: scheduled_at}] = all_enqueued()
 
-    {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
+    timestamp_dt = Timex.parse!(timestamp, "{ISO:Basic:Z}")
 
-    # as there is delay in setting and pulling the data from cache we ignore 1 second
-    # make it 2 seconds for flakyness
-    assert_in_delta(valid_till * 1000, cache_ttl, 2000)
+    assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
   end
 
   defp construct_s3_url(timestamp, valid_till) do
@@ -71,11 +88,11 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
 
   defp construct_metadata(timestamp, valid_till, url) do
     %{
-      image: construct_s3_url(timestamp, valid_till),
-      site: "Pleroma",
-      title: "Pleroma",
-      description: "Pleroma",
-      url: url
+      "image" => construct_s3_url(timestamp, valid_till),
+      "site" => "Pleroma",
+      "title" => "Pleroma",
+      "description" => "Pleroma",
+      "url" => url
     }
   end
 end
diff --git a/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs b/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs
new file mode 100644
index 0000000..770968d
--- /dev/null
+++ b/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.TTL.OpengraphTest do
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
+
+  test "OpenGraph TTL value is honored" do
+    url = "https://reddit.com/r/somepost"
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: ^url
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/reddit.html")}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
+    end)
+
+    Card.get_or_backfill_by_url(url)
+
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
+  end
+end
diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs
index 9064138..3fcb5c8 100755..100644
--- a/test/pleroma/web/rich_media/parser_test.exs
+++ b/test/pleroma/web/rich_media/parser_test.exs
@@ -3,95 +3,26 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.ParserTest do
-  use ExUnit.Case, async: true
+  use Pleroma.DataCase
 
   alias Pleroma.Web.RichMedia.Parser
 
-  setup do
-    Tesla.Mock.mock(fn
-      %{
-        method: :get,
-        url: "http://example.com/ogp"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/non-ogp"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/ogp-missing-title"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
-        }
-
-      %{
-        method: :get,
-        url: "http://example.com/twitter-card"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/oembed"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/oembed.json"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}
-
-      %{method: :get, url: "http://example.com/empty"} ->
-        %Tesla.Env{status: 200, body: "hello"}
-
-      %{method: :get, url: "http://example.com/malformed"} ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}
-
-      %{method: :get, url: "http://example.com/error"} ->
-        {:error, :overload}
-
-      %{
-        method: :head,
-        url: "http://example.com/huge-page"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
-        }
-
-      %{
-        method: :head,
-        url: "http://example.com/pdf-file"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
-        }
-
-      %{method: :head} ->
-        %Tesla.Env{status: 404, body: "", headers: []}
-    end)
+  import Tesla.Mock
 
-    :ok
+  setup do
+    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
   end
 
   test "returns error when no metadata present" do
-    assert {:error, _} = Parser.parse("http://example.com/empty")
+    assert {:error, _} = Parser.parse("https://example.com/empty")
   end
 
   test "doesn't just add a title" do
-    assert {:error, {:invalid_metadata, _}} = Parser.parse("http://example.com/non-ogp")
+    assert {:error, {:invalid_metadata, _}} = Parser.parse("https://example.com/non-ogp")
   end
 
   test "parses ogp" do
-    assert Parser.parse("http://example.com/ogp") ==
+    assert Parser.parse("https://example.com/ogp") ==
              {:ok,
               %{
                 "image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -99,12 +30,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
                 "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
                 "type" => "video.movie",
-                "url" => "http://example.com/ogp"
+                "url" => "https://example.com/ogp"
               }}
   end
 
   test "falls back to <title> when ogp:title is missing" do
-    assert Parser.parse("http://example.com/ogp-missing-title") ==
+    assert Parser.parse("https://example.com/ogp-missing-title") ==
              {:ok,
               %{
                 "image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -112,12 +43,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
                 "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
                 "type" => "video.movie",
-                "url" => "http://example.com/ogp-missing-title"
+                "url" => "https://example.com/ogp-missing-title"
               }}
   end
 
   test "parses twitter card" do
-    assert Parser.parse("http://example.com/twitter-card") ==
+    assert Parser.parse("https://example.com/twitter-card") ==
              {:ok,
               %{
                 "card" => "summary",
@@ -125,12 +56,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
                 "image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
                 "title" => "Small Island Developing States Photo Submission",
                 "description" => "View the album on Flickr.",
-                "url" => "http://example.com/twitter-card"
+                "url" => "https://example.com/twitter-card"
               }}
   end
 
   test "parses OEmbed and filters HTML tags" do
-    assert Parser.parse("http://example.com/oembed") ==
+    assert Parser.parse("https://example.com/oembed") ==
              {:ok,
               %{
                 "author_name" => "\u202E\u202D\u202Cbees\u202C",
@@ -150,7 +81,7 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
                 "thumbnail_width" => 150,
                 "title" => "Bacon Lollys",
                 "type" => "photo",
-                "url" => "http://example.com/oembed",
+                "url" => "https://example.com/oembed",
                 "version" => "1.0",
                 "web_page" => "https://www.flickr.com/photos/bees/2362225867/",
                 "web_page_short_url" => "https://flic.kr/p/4AK2sc",
@@ -159,18 +90,41 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
   end
 
   test "rejects invalid OGP data" do
-    assert {:error, _} = Parser.parse("http://example.com/malformed")
+    assert {:error, _} = Parser.parse("https://example.com/malformed")
   end
 
   test "returns error if getting page was not successful" do
-    assert {:error, :overload} = Parser.parse("http://example.com/error")
+    assert {:error, :overload} = Parser.parse("https://example.com/error")
   end
 
   test "does a HEAD request to check if the body is too large" do
-    assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
+    assert {:error, :body_too_large} = Parser.parse("https://example.com/huge-page")
   end
 
   test "does a HEAD request to check if the body is html" do
-    assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
+    assert {:error, {:content_type, _}} = Parser.parse("https://example.com/pdf-file")
+  end
+
+  test "refuses to crawl incomplete URLs" do
+    url = "example.com/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl malformed URLs" do
+    url = "example.com[]/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl URLs of private network from posts" do
+    [
+      "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO",
+      "https://10.111.10.1/notice/9kCP7V",
+      "https://172.16.32.40/notice/9kCP7V",
+      "https://192.168.10.40/notice/9kCP7V",
+      "https://pleroma.local/notice/9kCP7V"
+    ]
+    |> Enum.each(fn url ->
+      assert :error == Parser.parse(url)
+    end)
   end
 end
diff --git a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
index e84a4e5..e84a4e5 100755..100644
--- a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs