commit: e6f036b9adf91aa81afc676c13ccecf04359241c
parent: d80859731e50393851e6026ad301ed20d2b0db2a
Author: rinpatch <rinpatch@sdf.org>
Date: Wed, 26 Jun 2019 03:55:20 +0000
Merge branch 'issues/1016' into 'develop'
[#1016] add clear formatted text/ removes links with private ip
See merge request pleroma/pleroma!1335
Diffstat:
8 files changed, 102 insertions(+), 30 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [unreleased]
### Security
- Mastodon API: Fix display names not being sanitized
+- Rich media: Do not crawl private IP ranges
+
### Added
- Add a generic settings store for frontends / clients to use.
- Explicit addressing option for posting.
@@ -63,6 +65,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Configuration: `skip_thread_containment` option
- Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details.
- MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links.
+- Configuration: `ignore_hosts` option
+- Configuration: `ignore_tld` option
### Changed
- **Breaking:** bind to 127.0.0.1 instead of 0.0.0.0 by default
@@ -102,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Posts which are marked sensitive or tagged nsfw no longer have link previews.
- HTTP connection timeout is now set to 10 seconds.
- Respond with a 404 Not implemented JSON error message when requested API is not implemented
+- Rich Media: crawl only https URLs.
### Fixed
- Follow requests don't get 'stuck' anymore.
diff --git a/config/config.exs b/config/config.exs
@@ -330,7 +330,10 @@ config :pleroma, :mrf_keyword,
config :pleroma, :mrf_subchain, match_actor: %{}
-config :pleroma, :rich_media, enabled: true
+config :pleroma, :rich_media,
+ enabled: true,
+ ignore_hosts: [],
+ ignore_tld: ["local", "localdomain", "lan"]
config :pleroma, :media_proxy,
enabled: false,
diff --git a/config/test.exs b/config/test.exs
@@ -43,7 +43,11 @@ config :pleroma, Pleroma.Repo,
config :pbkdf2_elixir, rounds: 1
config :tesla, adapter: Tesla.Mock
-config :pleroma, :rich_media, enabled: false
+
+config :pleroma, :rich_media,
+ enabled: false,
+ ignore_hosts: [],
+ ignore_tld: ["local", "localdomain", "lan"]
config :web_push_encryption, :vapid_details,
subject: "mailto:administrator@example.com",
diff --git a/docs/config.md b/docs/config.md
@@ -417,6 +417,8 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
## :rich_media
* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
+* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"]
## :fetch_initial_posts
* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
@@ -4,27 +4,53 @@
defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Activity
+ alias Pleroma.Config
alias Pleroma.HTML
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
+ @spec validate_page_url(any()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
- if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do
- URI.parse(page_url) |> validate_page_url
- else
- :error
+ page_url
+ |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
+ |> parse_uri(page_url)
+ end
+
+ defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
+ when scheme == "https" and not is_nil(authority) do
+ cond do
+ host in Config.get([:rich_media, :ignore_hosts], []) ->
+ :error
+
+ get_tld(host) in Config.get([:rich_media, :ignore_tld], []) ->
+ :error
+
+ true ->
+ :ok
end
end
- defp validate_page_url(%URI{authority: nil}), do: :error
- defp validate_page_url(%URI{scheme: nil}), do: :error
- defp validate_page_url(%URI{}), do: :ok
defp validate_page_url(_), do: :error
+ defp parse_uri(true, url) do
+ url
+ |> URI.parse()
+ |> validate_page_url
+ end
+
+ defp parse_uri(_, _), do: :error
+
+ defp get_tld(host) do
+ host
+ |> String.split(".")
+ |> Enum.reverse()
+ |> hd
+ end
+
def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
- with true <- Pleroma.Config.get([:rich_media, :enabled]),
+ with true <- Config.get([:rich_media, :enabled]),
%Object{} = object <- Object.normalize(activity),
false <- object.data["sensitive"] || false,
{:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
@@ -757,6 +757,14 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
end
+ def get("https://example.com/ogp", _, _, _) do
+ {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
+ end
+
+ def get("https://pleroma.local/notice/9kCP7V", _, _, _) do
+ {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
+ end
+
def get("http://example.com/ogp-missing-data", _, _, _) do
{:ok,
%Tesla.Env{
@@ -765,6 +773,14 @@ defmodule HttpRequestMock do
}}
end
+ def get("https://example.com/ogp-missing-data", _, _, _) do
+ {:ok,
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/rich_media/ogp-missing-data.html")
+ }}
+ end
+
def get("http://example.com/malformed", _, _, _) do
{:ok,
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}}
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -312,7 +312,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
conn
|> assign(:user, user)
|> post("/api/v1/statuses", %{
- "status" => "http://example.com/ogp"
+ "status" => "https://example.com/ogp"
})
assert %{"id" => id, "card" => %{"title" => "The Rock"}} = json_response(conn, 200)
@@ -2557,7 +2557,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
end
test "returns rich-media card", %{conn: conn, user: user} do
- {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"})
+ {:ok, activity} = CommonAPI.post(user, %{"status" => "https://example.com/ogp"})
card_data = %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -2589,7 +2589,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
# works with private posts
{:ok, activity} =
- CommonAPI.post(user, %{"status" => "http://example.com/ogp", "visibility" => "direct"})
+ CommonAPI.post(user, %{"status" => "https://example.com/ogp", "visibility" => "direct"})
response_two =
conn
@@ -2601,7 +2601,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
end
test "replaces missing description with an empty string", %{conn: conn, user: user} do
- {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp-missing-data"})
+ {:ok, activity} =
+ CommonAPI.post(user, %{"status" => "https://example.com/ogp-missing-data"})
response =
conn
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
@@ -1,14 +1,19 @@
defmodule Pleroma.Web.RichMedia.HelpersTest do
use Pleroma.DataCase
+ alias Pleroma.Config
alias Pleroma.Object
alias Pleroma.Web.CommonAPI
+ alias Pleroma.Web.RichMedia.Helpers
import Pleroma.Factory
import Tesla.Mock
setup do
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+ rich_media = Config.get([:rich_media, :enabled])
+ on_exit(fn -> Config.put([:rich_media, :enabled], rich_media) end)
+
:ok
end
@@ -21,11 +26,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
"content_type" => "text/markdown"
})
- Pleroma.Config.put([:rich_media, :enabled], true)
+ Config.put([:rich_media, :enabled], true)
assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
- Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl malformed URLs" do
@@ -37,11 +40,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
"content_type" => "text/markdown"
})
- Pleroma.Config.put([:rich_media, :enabled], true)
+ Config.put([:rich_media, :enabled], true)
assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
- Pleroma.Config.put([:rich_media, :enabled], false)
end
test "crawls valid, complete URLs" do
@@ -49,16 +50,14 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
{:ok, activity} =
CommonAPI.post(user, %{
- "status" => "[test](http://example.com/ogp)",
+ "status" => "[test](https://example.com/ogp)",
"content_type" => "text/markdown"
})
- Pleroma.Config.put([:rich_media, :enabled], true)
+ Config.put([:rich_media, :enabled], true)
- assert %{page_url: "http://example.com/ogp", rich_media: _} =
+ assert %{page_url: "https://example.com/ogp", rich_media: _} =
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
- Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl URLs from posts marked sensitive" do
@@ -74,11 +73,9 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
assert object.data["sensitive"]
- Pleroma.Config.put([:rich_media, :enabled], true)
+ Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-
- Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl URLs from posts tagged NSFW" do
@@ -93,10 +90,28 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
assert object.data["sensitive"]
- Pleroma.Config.put([:rich_media, :enabled], true)
+ Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+ end
+
+ test "refuses to crawl URLs of private network from posts" do
+ user = insert(:user)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{"status" => "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
+
+ {:ok, activity2} = CommonAPI.post(user, %{"status" => "https://10.111.10.1/notice/9kCP7V"})
+ {:ok, activity3} = CommonAPI.post(user, %{"status" => "https://172.16.32.40/notice/9kCP7V"})
+ {:ok, activity4} = CommonAPI.post(user, %{"status" => "https://192.168.10.40/notice/9kCP7V"})
+ {:ok, activity5} = CommonAPI.post(user, %{"status" => "https://pleroma.local/notice/9kCP7V"})
+
+ Config.put([:rich_media, :enabled], true)
- Pleroma.Config.put([:rich_media, :enabled], false)
+ assert %{} = Helpers.fetch_data_for_activity(activity)
+ assert %{} = Helpers.fetch_data_for_activity(activity2)
+ assert %{} = Helpers.fetch_data_for_activity(activity3)
+ assert %{} = Helpers.fetch_data_for_activity(activity4)
+ assert %{} = Helpers.fetch_data_for_activity(activity5)
end
end
