commit: 2498e569f12694439b6f99d0730f6fb36301c454
parent 89c595b772eaaa8809f5339d708d7dc22e51b662
Author: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 11 Oct 2020 22:34:48 +0300
Merge remote-tracking branch 'remotes/origin/develop' into ostatus-controller-no-auth-check-on-non-federating-instances
Diffstat:
13 files changed, 190 insertions(+), 90 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed
- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
+- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
- Search: Users are now findable by their urls.
- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
diff --git a/config/config.exs b/config/config.exs
@@ -677,7 +677,18 @@ config :pleroma, :rate_limit,
config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+config :pleroma, Pleroma.Plugs.RemoteIp,
+ enabled: true,
+ headers: ["x-forwarded-for"],
+ proxies: [],
+ reserved: [
+ "127.0.0.0/8",
+ "::1/128",
+ "fc00::/7",
+ "10.0.0.0/8",
+ "172.16.0.0/12",
+ "192.168.0.0/16"
+ ]
config :pleroma, :static_fe, enabled: false
diff --git a/config/description.exs b/config/description.exs
@@ -44,11 +44,13 @@ frontend_options = [
},
%{
key: "git",
+ label: "Git Repository URL",
type: :string,
description: "URL of the git repository of the frontend"
},
%{
key: "build_url",
+ label: "Build URL",
type: :string,
description:
"Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@@ -56,6 +58,7 @@ frontend_options = [
},
%{
key: "build_dir",
+ label: "Build directory",
type: :string,
description: "The directory inside the zip file "
}
@@ -3262,20 +3265,22 @@ config :pleroma, :config_description, [
%{
key: :headers,
type: {:list, :string},
- description:
- "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
+ description: """
+ A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
+ """
},
%{
key: :proxies,
type: {:list, :string},
description:
- "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
+ "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
},
%{
key: :reserved,
type: {:list, :string},
- description:
- "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
+ description: """
+ A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
+ """
}
]
},
@@ -3681,9 +3686,7 @@ config :pleroma, :config_description, [
type: :map,
description:
"A map containing available frontends and parameters for their installation.",
- children: [
- frontend_options
- ]
+ children: frontend_options
}
]
},
diff --git a/docs/administration/backup.md b/docs/administration/backup.md
@@ -5,20 +5,24 @@
1. Stop the Pleroma service.
2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
-4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
+4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
5. Restart the Pleroma service.
## Restore/Move
-1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
+1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
2. Stop the Pleroma service.
3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
4. Copy the above mentioned files back to their original position.
-5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
-6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
-7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
-8. Restart the Pleroma service.
-9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;'`
+6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`.
+
+ Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
+
+7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+9. Restart the Pleroma service.
+10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
@@ -426,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
Available options:
* `enabled` - Enable/disable the plug. Defaults to `false`.
-* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
-* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
-* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
+* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
+* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
+* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
### :rate_limit
diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex
@@ -88,7 +88,7 @@ defmodule Pleroma.Emails.AdminEmail do
html_body = """
<p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
<blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
- <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a>
+ <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
"""
new()
diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex
@@ -7,48 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do
This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
"""
+ alias Pleroma.Config
import Plug.Conn
@behaviour Plug
- @headers ~w[
- x-forwarded-for
- ]
-
- # https://en.wikipedia.org/wiki/Localhost
- # https://en.wikipedia.org/wiki/Private_network
- @reserved ~w[
- 127.0.0.0/8
- ::1/128
- fc00::/7
- 10.0.0.0/8
- 172.16.0.0/12
- 192.168.0.0/16
- ]
-
def init(_), do: nil
def call(%{remote_ip: original_remote_ip} = conn, _) do
- config = Pleroma.Config.get(__MODULE__, [])
-
- if Keyword.get(config, :enabled, false) do
- %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
+ if Config.get([__MODULE__, :enabled]) do
+ %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts())
assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
else
conn
end
end
- defp remote_ip_opts(config) do
- headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
- reserved = Keyword.get(config, :reserved, @reserved)
+ defp remote_ip_opts do
+ headers = Config.get([__MODULE__, :headers], []) |> MapSet.new()
+ reserved = Config.get([__MODULE__, :reserved], [])
proxies =
- config
- |> Keyword.get(:proxies, [])
+ Config.get([__MODULE__, :proxies], [])
|> Enum.concat(reserved)
- |> Enum.map(&InetCidr.parse/1)
+ |> Enum.map(&maybe_add_cidr/1)
{headers, proxies}
end
+
+ defp maybe_add_cidr(proxy) when is_binary(proxy) do
+ proxy =
+ cond do
+ "/" in String.codepoints(proxy) -> proxy
+ InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
+ InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
+ end
+
+ InetCidr.parse(proxy, true)
+ end
end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -790,7 +790,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
[activity, object] in query,
where:
fragment(
- "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
+ """
+ ?->>'type' != 'Create' -- This isn't a Create
+ OR ?->>'inReplyTo' is null -- this isn't a reply
+ OR ? && array_remove(?, ?) -- The recipient is us or one of our friends,
+ -- unless they are the author (because authors
+ -- are also part of the recipients). This leads
+ -- to a bug that self-replies by friends won't
+ -- show up.
+ OR ? = ? -- The actor is us
+ """,
+ activity.data,
object.data,
^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
activity.recipients,
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
@@ -57,7 +57,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
def fetch_data_for_object(object) do
with true <- Config.get([:rich_media, :enabled]),
- false <- object.data["sensitive"] || false,
{:ok, page_url} <-
HTML.extract_first_external_url_from_object(object),
:ok <- validate_page_url(page_url),
diff --git a/test/emails/admin_email_test.exs b/test/emails/admin_email_test.exs
@@ -63,7 +63,7 @@ defmodule Pleroma.Emails.AdminEmailTest do
assert res.html_body == """
<p>New account for review: <a href="#{account_url}">@#{account.nickname}</a></p>
<blockquote>Plz let me in</blockquote>
- <a href="http://localhost:4001/pleroma/admin">Visit AdminFE</a>
+ <a href="http://localhost:4001/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
"""
end
end
diff --git a/test/plugs/remote_ip_test.exs b/test/plugs/remote_ip_test.exs
@@ -3,13 +3,27 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.RemoteIpTest do
- use ExUnit.Case, async: true
+ use ExUnit.Case
use Plug.Test
alias Pleroma.Plugs.RemoteIp
- import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2]
- setup do: clear_config(RemoteIp)
+ import Pleroma.Tests.Helpers, only: [clear_config: 2]
+
+ setup do:
+ clear_config(RemoteIp,
+ enabled: true,
+ headers: ["x-forwarded-for"],
+ proxies: [],
+ reserved: [
+ "127.0.0.0/8",
+ "::1/128",
+ "fc00::/7",
+ "10.0.0.0/8",
+ "172.16.0.0/12",
+ "192.168.0.0/16"
+ ]
+ )
test "disabled" do
Pleroma.Config.put(RemoteIp, enabled: false)
@@ -25,8 +39,6 @@ defmodule Pleroma.Plugs.RemoteIpTest do
end
test "enabled" do
- Pleroma.Config.put(RemoteIp, enabled: true)
-
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "1.1.1.1")
@@ -54,8 +66,6 @@ defmodule Pleroma.Plugs.RemoteIpTest do
end
test "custom proxies" do
- Pleroma.Config.put(RemoteIp, enabled: true)
-
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2")
@@ -72,4 +82,27 @@ defmodule Pleroma.Plugs.RemoteIpTest do
assert conn.remote_ip == {1, 1, 1, 1}
end
+
+ test "proxies set without CIDR format" do
+ Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"])
+
+ conn =
+ conn(:get, "/")
+ |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1")
+ |> RemoteIp.call(nil)
+
+ assert conn.remote_ip == {1, 1, 1, 1}
+ end
+
+ test "proxies set `nonsensical` CIDR" do
+ Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"])
+ Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"])
+
+ conn =
+ conn(:get, "/")
+ |> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1")
+ |> RemoteIp.call(nil)
+
+ assert conn.remote_ip == {1, 1, 1, 1}
+ end
end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
@@ -2177,4 +2177,84 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
assert user.nickname == orig_user.nickname
end
end
+
+ describe "reply filtering" do
+ test "`following` still contains announcements by friends" do
+ user = insert(:user)
+ followed = insert(:user)
+ not_followed = insert(:user)
+
+ User.follow(user, followed)
+
+ {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
+
+ {:ok, not_followed_to_followed} =
+ CommonAPI.post(not_followed, %{
+ status: "Also hello",
+ in_reply_to_status_id: followed_post.id
+ })
+
+ {:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed)
+
+ params =
+ %{}
+ |> Map.put(:type, ["Create", "Announce"])
+ |> Map.put(:blocking_user, user)
+ |> Map.put(:muting_user, user)
+ |> Map.put(:reply_filtering_user, user)
+ |> Map.put(:reply_visibility, "following")
+ |> Map.put(:announce_filtering_user, user)
+ |> Map.put(:user, user)
+
+ activities =
+ [user.ap_id | User.following(user)]
+ |> ActivityPub.fetch_activities(params)
+
+ followed_post_id = followed_post.id
+ retoot_id = retoot.id
+
+ assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities
+
+ assert length(activities) == 2
+ end
+
+ # This test is skipped because, while this is the desired behavior,
+ # there seems to be no good way to achieve it with the method that
+ # we currently use for detecting to who a reply is directed.
+ # This is a TODO and should be fixed by a later rewrite of the code
+ # in question.
+ @tag skip: true
+ test "`following` still contains self-replies by friends" do
+ user = insert(:user)
+ followed = insert(:user)
+ not_followed = insert(:user)
+
+ User.follow(user, followed)
+
+ {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
+ {:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"})
+
+ {:ok, _followed_to_not_followed} =
+ CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id})
+
+ {:ok, _followed_self_reply} =
+ CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id})
+
+ params =
+ %{}
+ |> Map.put(:type, ["Create", "Announce"])
+ |> Map.put(:blocking_user, user)
+ |> Map.put(:muting_user, user)
+ |> Map.put(:reply_filtering_user, user)
+ |> Map.put(:reply_visibility, "following")
+ |> Map.put(:announce_filtering_user, user)
+ |> Map.put(:user, user)
+
+ activities =
+ [user.ap_id | User.following(user)]
+ |> ActivityPub.fetch_activities(params)
+
+ assert length(activities) == 2
+ end
+ end
end
diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs
@@ -64,41 +64,6 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
- test "refuses to crawl URLs from posts marked sensitive" do
- user = insert(:user)
-
- {:ok, activity} =
- CommonAPI.post(user, %{
- status: "http://example.com/ogp",
- sensitive: true
- })
-
- %Object{} = object = Object.normalize(activity)
-
- assert object.data["sensitive"]
-
- Config.put([:rich_media, :enabled], true)
-
- assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
- end
-
- test "refuses to crawl URLs from posts tagged NSFW" do
- user = insert(:user)
-
- {:ok, activity} =
- CommonAPI.post(user, %{
- status: "http://example.com/ogp #nsfw"
- })
-
- %Object{} = object = Object.normalize(activity)
-
- assert object.data["sensitive"]
-
- Config.put([:rich_media, :enabled], true)
-
- assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
- end
-
test "refuses to crawl URLs of private network from posts" do
user = insert(:user)
