commit: 18a91d85e59eb67a2d58a28aae732c34b75ebe97 parent 2a475622eea5550c9ab455c4e86212fc09ee9c58 Author: Egor Kislitsyn <egor@kislitsyn.com> Date: Mon, 19 Oct 2020 15:31:56 +0400 Merge branch 'develop' into feature/local-only-scopeDiffstat:
27 files changed, 223 insertions(+), 32 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -27,6 +27,8 @@ erl_crash.dump # variables. /config/*.secret.exs /config/generated_config.exs +/config/*.env + # Database setup file, some may forget to delete it /config/setup_db.psql diff --git a/CHANGELOG.md b/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). +- Pleroma API: Importing the mutes users from CSV files. +- Experimental websocket-based federation between Pleroma instances. - Support for local-only statuses ### Changed @@ -24,11 +27,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`. -### Added -- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). -- Pleroma API: Importing the mutes users from CSV files. -- Experimental websocket-based federation between Pleroma instances. - <details> <summary>API Changes</summary> diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md @@ -40,3 +40,5 @@ If any of the options are left unspecified, you will be prompted interactively. - `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data - `--anonymize-uploads <Y|N>` - randomize uploaded filenames - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames +- `--skip-release-env` - skip generation the release environment file +- `--release-env-file` - release environment file path diff --git a/docs/administration/CLI_tasks/release_environments.md b/docs/administration/CLI_tasks/release_environments.md @@ -0,0 +1,9 @@ +# Generate release environment file + +```sh tab="OTP" + ./bin/pleroma_ctl release_env gen +``` + +```sh tab="From Source" +mix pleroma.release_env gen +``` diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md @@ -101,6 +101,7 @@ sudo -Hu pleroma mix deps.get mv config/{generated_config.exs,prod.secret.exs} ``` + * The previous command creates also the file `config/setup_db.psql`, with which you can create the database: ```shell @@ -181,6 +182,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se ``` * Edit the service file and make sure that all paths fit your installation +* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen` * Enable and start `pleroma.service`: ```shell diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md @@ -149,6 +149,9 @@ chown -R pleroma /etc/pleroma # Run the config generator su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql" +# Run the environment file generator. +su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen" + # Create the postgres database su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql" @@ -159,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate" # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" # Start the instance to verify that everything is working as expected -su pleroma -s $SHELL -lc "./bin/pleroma daemon" +su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon" # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly sleep 20 && curl http://localhost:4000/api/v1/instance @@ -311,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre ## Questions Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. - diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma @@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid" directory=/opt/pleroma healthcheck_delay=60 healthcheck_timer=30 +export $(cat /opt/pleroma/config/pleroma.env) : ${pleroma_port:-4000} diff --git a/installation/pleroma.service b/installation/pleroma.service @@ -17,6 +17,8 @@ Environment="MIX_ENV=prod" Environment="HOME=/var/lib/pleroma" ; Path to the folder containing the Pleroma installation. WorkingDirectory=/opt/pleroma +; Path to the environment file. the file contains RELEASE_COOKIE and etc +EnvironmentFile=/opt/pleroma/config/pleroma.env ; Path to the Mix binary. ExecStart=/usr/bin/mix phx.server diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex @@ -36,7 +36,9 @@ defmodule Mix.Tasks.Pleroma.Instance do listen_port: :string, strip_uploads: :string, anonymize_uploads: :string, - dedupe_uploads: :string + dedupe_uploads: :string, + skip_release_env: :boolean, + release_env_file: :string ], aliases: [ o: :output, @@ -241,6 +243,24 @@ defmodule Mix.Tasks.Pleroma.Instance do write_robots_txt(static_dir, indexable, template_dir) + if Keyword.get(options, :skip_release_env, false) do + shell_info(""" + Release environment file is skip. Please generate the release env file before start. + `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen` + """) + else + shell_info("Generation the environment file:") + + release_env_args = + with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do + ["gen", "--path", path] + else + _ -> ["gen"] + end + + Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args) + end + shell_info( "\n All files successfully written! Refer to the installation instructions for your platform for next steps." ) diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex @@ -0,0 +1,76 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.ReleaseEnv do + use Mix.Task + import Mix.Pleroma + + @shortdoc "Generate Pleroma environment file." + @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md") + + def run(["gen" | rest]) do + {options, [], []} = + OptionParser.parse( + rest, + strict: [ + force: :boolean, + path: :string + ], + aliases: [ + p: :path, + f: :force + ] + ) + + file_path = + get_option( + options, + :path, + "Environment file path", + "./config/pleroma.env" + ) + + env_path = Path.expand(file_path) + + proceed? = + if File.exists?(env_path) do + get_option( + options, + :force, + "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)", + "n" + ) === "y" + else + true + end + + if proceed? do + case do_generate(env_path) do + {:error, reason} -> + shell_error( + File.Error.message(%{action: "write to file", reason: reason, path: env_path}) + ) + + _ -> + shell_info("\nThe file generated: #{env_path}.\n") + + shell_info(""" + WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable. + Example: + chmod 0444 #{file_path} + chattr +i #{file_path} + """) + end + else + shell_info("\nThe file is exist. #{env_path}.\n") + end + end + + def do_generate(path) do + content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}" + + File.mkdir_p!(Path.dirname(path)) + File.write(path, content) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex @@ -136,7 +136,7 @@ defmodule Pleroma.User do field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: []) field(:raw_fields, {:array, :map}, default: []) - field(:discoverable, :boolean, default: false) + field(:is_discoverable, :boolean, default: false) field(:invisible, :boolean, default: false) field(:allow_following_move, :boolean, default: true) field(:skip_thread_containment, :boolean, default: false) @@ -448,7 +448,7 @@ defmodule Pleroma.User do :follower_count, :fields, :following_count, - :discoverable, + :is_discoverable, :invisible, :actor_type, :also_known_as, @@ -495,7 +495,7 @@ defmodule Pleroma.User do :fields, :raw_fields, :pleroma_settings_store, - :discoverable, + :is_discoverable, :actor_type, :also_known_as, :accepts_chat_messages @@ -1618,7 +1618,7 @@ defmodule Pleroma.User do pleroma_settings_store: %{}, fields: [], raw_fields: [], - discoverable: false, + is_discoverable: false, also_known_as: [] }) end diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex @@ -164,7 +164,7 @@ defmodule Pleroma.User.Search do end defp filter_discoverable_users(query) do - from(q in query, where: q.discoverable == true) + from(q in query, where: q.is_discoverable == true) end defp filter_internal_users(query) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -827,7 +827,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do query = from([activity] in query, where: fragment("not (? = ANY(?))", activity.actor, ^mutes), - where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes) + where: + fragment( + "not (?->'to' \\?| ?) or ? = ?", + activity.data, + ^mutes, + activity.actor, + ^user.ap_id + ) ) unless opts[:skip_preload] do @@ -1232,7 +1239,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do capabilities = data["capabilities"] || %{} accepts_chat_messages = capabilities["acceptsChatMessages"] data = Transmogrifier.maybe_fix_user_object(data) - discoverable = data["discoverable"] || false + is_discoverable = data["discoverable"] || false invisible = data["invisible"] || false actor_type = data["type"] || "Person" @@ -1258,7 +1265,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fields: fields, emoji: emojis, is_locked: is_locked, - discoverable: discoverable, + is_discoverable: is_discoverable, invisible: invisible, avatar: avatar, name: data["name"], diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -110,7 +110,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "endpoints" => endpoints, "attachment" => fields, "tag" => emoji_tags, - "discoverable" => user.discoverable, + "discoverable" => user.is_discoverable, "capabilities" => capabilities } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -185,7 +185,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do :show_role, :skip_thread_containment, :allow_following_move, - :discoverable, :accepts_chat_messages ] |> Enum.reduce(%{}, fn key, acc -> @@ -210,6 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end) |> Maps.put_if_present(:actor_type, params[:actor_type]) |> Maps.put_if_present(:is_locked, params[:locked]) + |> Maps.put_if_present(:is_discoverable, params[:discoverable]) # What happens here: # diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -261,7 +261,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do sensitive: false, fields: user.raw_fields, pleroma: %{ - discoverable: user.discoverable, + discoverable: user.is_discoverable, actor_type: user.actor_type } }, diff --git a/lib/pleroma/web/metadata/providers/restrict_indexing.ex b/lib/pleroma/web/metadata/providers/restrict_indexing.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do """ @impl true - def build_tags(%{user: %{local: true, discoverable: true}}), do: [] + def build_tags(%{user: %{local: true, is_discoverable: true}}), do: [] def build_tags(_) do [ diff --git a/priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs b/priv/repo/migrations/20201013144052_refactor_discoverable_user_field.exs @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.RefactorDiscoverableUserField do + use Ecto.Migration + + def up do + execute("ALTER TABLE users RENAME COLUMN discoverable TO is_discoverable;") + end + + def down do + execute("ALTER TABLE users RENAME COLUMN is_discoverable TO discoverable;") + end +end diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs @@ -5,6 +5,8 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do use ExUnit.Case + @release_env_file "./test/pleroma.test.env" + setup do File.mkdir_p!(tmp_path()) @@ -16,6 +18,8 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do File.rm_rf(Path.join(static_dir, "robots.txt")) end + if File.exists?(@release_env_file), do: File.rm_rf(@release_env_file) + Pleroma.Config.put([:instance, :static_dir], static_dir) end) @@ -69,7 +73,9 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do "--dedupe-uploads", "n", "--anonymize-uploads", - "n" + "n", + "--release-env-file", + @release_env_file ]) end @@ -91,6 +97,9 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.exists?(Path.expand("./test/instance/static/robots.txt")) + assert File.exists?(@release_env_file) + + assert File.read!(@release_env_file) =~ ~r/^RELEASE_COOKIE=.*/ end defp generated_setup_psql do diff --git a/test/mix/tasks/pleroma/release_env_test.exs b/test/mix/tasks/pleroma/release_env_test.exs @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do + use ExUnit.Case + import ExUnit.CaptureIO, only: [capture_io: 1] + + @path "config/pleroma.test.env" + + def do_clean do + if File.exists?(@path) do + File.rm_rf(@path) + end + end + + setup do + do_clean() + on_exit(fn -> do_clean() end) + :ok + end + + test "generate pleroma.env" do + assert capture_io(fn -> + Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"]) + end) =~ "The file generated" + + assert File.read!(@path) =~ "RELEASE_COOKIE=" + end +end diff --git a/test/pleroma/user_search_test.exs b/test/pleroma/user_search_test.exs @@ -66,7 +66,7 @@ defmodule Pleroma.UserSearchTest do end test "excludes users when discoverable is false" do - insert(:user, %{nickname: "john 3000", discoverable: false}) + insert(:user, %{nickname: "john 3000", is_discoverable: false}) insert(:user, %{nickname: "john 3001"}) users = User.search("john") diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs @@ -1467,7 +1467,7 @@ defmodule Pleroma.UserTest do pleroma_settings_store: %{"q" => "x"}, fields: [%{"gg" => "qq"}], raw_fields: [%{"gg" => "qq"}], - discoverable: true, + is_discoverable: true, also_known_as: ["https://lol.olo/users/loll"] }) @@ -1509,7 +1509,7 @@ defmodule Pleroma.UserTest do pleroma_settings_store: %{}, fields: [], raw_fields: [], - discoverable: false, + is_discoverable: false, also_known_as: [] } = user end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -752,6 +752,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do refute repeat_activity in activities end + test "returns your own posts regardless of mute" do + user = insert(:user) + muted = insert(:user) + + {:ok, muted_post} = CommonAPI.post(muted, %{status: "Im stupid"}) + + {:ok, reply} = + CommonAPI.post(user, %{status: "I'm muting you", in_reply_to_status_id: muted_post.id}) + + {:ok, _} = User.mute(user, muted) + + [activity] = ActivityPub.fetch_activities([], %{muting_user: user, skip_preload: true}) + + assert activity.id == reply.id + end + test "doesn't return muted activities" do activity_one = insert(:note_activity) activity_two = insert(:note_activity) diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs @@ -180,7 +180,7 @@ defmodule Pleroma.Web.AdminAPI.SearchTest do test "it returns non-discoverable users" do insert(:user) - insert(:user, discoverable: false) + insert(:user, is_discoverable: false) {:ok, _results, total} = Search.user() diff --git a/test/pleroma/web/metadata/providers/restrict_indexing_test.exs b/test/pleroma/web/metadata/providers/restrict_indexing_test.exs @@ -14,13 +14,13 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexingTest do test "for local user" do assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{ - user: %Pleroma.User{local: true, discoverable: true} + user: %Pleroma.User{local: true, is_discoverable: true} }) == [] end test "for local user when discoverable is false" do assert Pleroma.Web.Metadata.Providers.RestrictIndexing.build_tags(%{ - user: %Pleroma.User{local: true, discoverable: false} + user: %Pleroma.User{local: true, is_discoverable: false} }) == [{:meta, [name: "robots", content: "noindex, noarchive"], []}] end end diff --git a/test/pleroma/web/metadata_test.exs b/test/pleroma/web/metadata_test.exs @@ -16,14 +16,14 @@ defmodule Pleroma.Web.MetadataTest do end test "for local user" do - user = insert(:user, discoverable: false) + user = insert(:user, is_discoverable: false) assert Pleroma.Web.Metadata.build_tags(%{user: user}) =~ "<meta content=\"noindex, noarchive\" name=\"robots\">" end test "for local user set to discoverable" do - user = insert(:user, discoverable: true) + user = insert(:user, is_discoverable: true) refute Pleroma.Web.Metadata.build_tags(%{user: user}) =~ "<meta content=\"noindex, noarchive\" name=\"robots\">" @@ -33,14 +33,14 @@ defmodule Pleroma.Web.MetadataTest do describe "no metadata for private instances" do test "for local user set to discoverable" do clear_config([:instance, :public], false) - user = insert(:user, bio: "This is my secret fedi account bio", discoverable: true) + user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: true) assert "" = Pleroma.Web.Metadata.build_tags(%{user: user}) end test "search exclusion metadata is included" do clear_config([:instance, :public], false) - user = insert(:user, bio: "This is my secret fedi account bio", discoverable: false) + user = insert(:user, bio: "This is my secret fedi account bio", is_discoverable: false) assert ~s(<meta content="noindex, noarchive" name="robots">) == Pleroma.Web.Metadata.build_tags(%{user: user}) diff --git a/test/support/factory.ex b/test/support/factory.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Factory do nickname: sequence(:nickname, &"nick#{&1}"), password_hash: Pbkdf2.hash_pwd_salt("test"), bio: sequence(:bio, &"Tester Number #{&1}"), - discoverable: true, + is_discoverable: true, last_digest_emailed_at: NaiveDateTime.utc_now(), last_refreshed_at: NaiveDateTime.utc_now(), notification_settings: %Pleroma.User.NotificationSetting{},