commit: cc42b50c5bbbeee066d3d26907eebb488ec6aa3d parent fdc3cbb8cbefad2161cc408b4440420d6e4b2a88 Author: Lain Soykaf <lain@lain.com> Date: Tue, 28 May 2024 16:51:19 +0400 Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into pleroma-feature/akkoma-prune-old-postsDiffstat:
173 files changed, 4366 insertions(+), 743 deletions(-)diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml@@ -1,8 +1,8 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 variables: &global_variables # Only used for the release - ELIXIR_VER: 1.12.3 + ELIXIR_VER: 1.13.4 POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -72,7 +72,7 @@ check-changelog: tags: - amd64 -build-1.12.3: +build-1.13.4: extends: - .build_changes_policy - .using-ci-base @@ -85,7 +85,7 @@ build-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: build - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 allow_failure: true script: - mix compile --forcediff --git a/Dockerfile b/Dockerfile@@ -1,7 +1,7 @@ ARG ELIXIR_IMG=hexpm/elixir -ARG ELIXIR_VER=1.12.3 -ARG ERLANG_VER=24.2.1 -ARG ALPINE_VER=3.17.0 +ARG ELIXIR_VER=1.13.4 +ARG ERLANG_VER=24.3.4.15 +ARG ALPINE_VER=3.17.5 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as builddiff --git a/changelog.d/3904.security b/changelog.d/3904.security@@ -0,0 +1 @@ +HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation.diff --git a/changelog.d/3907.skip b/changelog.d/3907.skipdiff --git a/changelog.d/add-ipfs-upload.add b/changelog.d/add-ipfs-upload.add@@ -0,0 +1 @@ +Uploader: Add support for uploading attachments using IPFSdiff --git a/changelog.d/add-nsfw-mrf.add b/changelog.d/add-nsfw-mrf.add@@ -0,0 +1 @@ +Add NSFW-detecting MRFdiff --git a/changelog.d/add-rbl-mrf.add b/changelog.d/add-rbl-mrf.add@@ -0,0 +1 @@ +Add DNSRBL MRFdiff --git a/changelog.d/anti-mentionspam-mrf.add b/changelog.d/anti-mentionspam-mrf.add@@ -0,0 +1 @@ +Add Anti-mention Spam MRF backported from Rebaseddiff --git a/changelog.d/api-docs-2.skip b/changelog.d/api-docs-2.skipdiff --git a/changelog.d/auth-fetch-exception.add b/changelog.d/auth-fetch-exception.add@@ -0,0 +1 @@ +HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration +\ No newline at end of filediff --git a/changelog.d/authorized-fetch-rejections.add b/changelog.d/authorized-fetch-rejections.add@@ -0,0 +1 @@ +Add an option to reject certain domains when authorized fetch is enabled.diff --git a/changelog.d/bump-elixir.change b/changelog.d/bump-elixir.change@@ -0,0 +1 @@ +Elixir 1.13 is the minimum required version.diff --git a/changelog.d/familiar-followers.add b/changelog.d/familiar-followers.add@@ -0,0 +1 @@ +Implement `/api/v1/accounts/familiar_followers` +\ No newline at end of filediff --git a/changelog.d/fix-webfinger-spoofing.security b/changelog.d/fix-webfinger-spoofing.security@@ -0,0 +1 @@ +Fix webfinger spoofing.diff --git a/changelog.d/instance-rules.add b/changelog.d/instance-rules.add@@ -0,0 +1 @@ +Add instance rules +\ No newline at end of filediff --git a/changelog.d/logger-metadata.add b/changelog.d/logger-metadata.add@@ -0,0 +1 @@ +Logger metadata is now attached to some logs to help with troubleshooting and analysisdiff --git a/changelog.d/mark-read.fix b/changelog.d/mark-read.fix@@ -0,0 +1 @@ +The query for marking notifications as read has been simplifieddiff --git a/changelog.d/mastodon_api_v2.add b/changelog.d/mastodon_api_v2.add@@ -0,0 +1 @@ +Add new parameters to /api/v2/instance: configuration[accounts][max_pinned_statuses] and configuration[statuses][characters_reserved_per_url]diff --git a/changelog.d/mediaproxy-http.fix b/changelog.d/mediaproxy-http.fix@@ -0,0 +1 @@ +Ensure MediaProxy HTTP requests obey all the defined connection settingsdiff --git a/changelog.d/missing-mrfs.add b/changelog.d/missing-mrfs.add@@ -0,0 +1 @@ +Startup detection for configured MRF modules that are missing or incorrectly defineddiff --git a/changelog.d/oban-queues.change b/changelog.d/oban-queues.change@@ -0,0 +1 @@ +Oban queues have refactored to simplify the queue designdiff --git a/changelog.d/pools.change b/changelog.d/pools.change@@ -0,0 +1 @@ +HTTP connection pool adjustmentsdiff --git a/changelog.d/prometheus-docs.change b/changelog.d/prometheus-docs.change@@ -0,0 +1 @@ +Update the documentation for configuring Prometheus metrics.diff --git a/changelog.d/promexdocs.add b/changelog.d/promexdocs.add@@ -0,0 +1 @@ +PromEx documentationdiff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add@@ -0,0 +1 @@ +Add Qdrant/OpenAI embedding searchdiff --git a/changelog.d/realpath-over-readlink.fix b/changelog.d/realpath-over-readlink.fix@@ -0,0 +1 @@ +pleroma_ctl: Use realpath(1) instead of readlink(1)diff --git a/changelog.d/reply-to-deleted.change b/changelog.d/reply-to-deleted.change@@ -0,0 +1 @@ +A 422 error is returned when attempting to reply to a deleted statusdiff --git a/changelog.d/richmediattl.fix b/changelog.d/richmediattl.fix@@ -0,0 +1 @@ +Parsing of RichMedia TTLs for Amazon URLs when query parameters are nildiff --git a/changelog.d/search-healthcheck.add b/changelog.d/search-healthcheck.add@@ -0,0 +1 @@ +Monitoring of search backend health to control the processing of jobs in the search indexing Oban queuediff --git a/changelog.d/show-reposter-replies.add b/changelog.d/show-reposter-replies.add@@ -0,0 +1 @@ +Display reposted replies with exclude_replies: true +\ No newline at end of filediff --git a/changelog.d/status-notification-type.add b/changelog.d/status-notification-type.add@@ -0,0 +1 @@ +Add "status" notification type +\ No newline at end of filediff --git a/changelog.d/support-honk-image-summaries.add b/changelog.d/support-honk-image-summaries.add@@ -0,0 +1 @@ +Support honk-style attachment summaries as alt-text.diff --git a/changelog.d/web_push_filtered.fix b/changelog.d/web_push_filtered.fix@@ -0,0 +1 @@ +Web Push notifications are no longer generated for muted/blocked threads and users.diff --git a/changelog.d/webfinger-validation.fix b/changelog.d/webfinger-validation.fix@@ -0,0 +1 @@ +Fix validate_webfinger when running a different domain for Webfinger +\ No newline at end of filediff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13/Dockerfile@@ -0,0 +1,8 @@ +FROM elixir:1.13.4-otp-24 + +# Single RUN statement, otherwise intermediate images are created +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run +RUN apt-get update &&\ + apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ + mix local.hex --force &&\ + mix local.rebar --forcediff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh@@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh@@ -1 +1 @@ -docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .diff --git a/config/config.exs b/config/config.exs@@ -82,6 +82,10 @@ config :ex_aws, :s3, # region: "us-east-1", # may be required for Amazon AWS scheme: "https://" +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: nil, + get_gateway_url: nil + config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], pack_extensions: [".png", ".gif"], @@ -131,13 +135,13 @@ config :pleroma, Pleroma.Web.Endpoint, config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:request_id] + metadata: [:actor, :path, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:request_id] + metadata: [:actor, :path, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], @@ -188,6 +192,7 @@ config :pleroma, :instance, allow_relay: true, public: true, quarantined_instances: [], + rejected_instances: [], static_dir: "instance/static/", allowed_post_formats: [ "text/plain", @@ -406,11 +411,23 @@ config :pleroma, :mrf_vocabulary, accept: [], reject: [] +config :pleroma, :mrf_dnsrbl, + nameserver: "127.0.0.1", + port: 53, + zone: "bl.pleroma.com" + # threshold of 7 days config :pleroma, :mrf_object_age, threshold: 604_800, actions: [:delist, :strip_followers] +config :pleroma, :mrf_nsfw_api, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + config :pleroma, :mrf_follow_bot, follower_nickname: nil config :pleroma, :mrf_inline_quote, template: "<bdi>RT:</bdi> {url}" @@ -419,6 +436,8 @@ config :pleroma, :mrf_force_mention, mention_parent: true, mention_quoted: true +config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000 + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], @@ -501,7 +520,8 @@ config :pleroma, :http_security, sts: false, sts_max_age: 31_536_000, ct_max_age: 2_592_000, - referrer_policy: "same-origin" + referrer_policy: "same-origin", + allow_unsafe_eval: false config :cors_plug, max_age: 86_400, @@ -563,24 +583,14 @@ config :pleroma, Oban, log: false, queues: [ activity_expiration: 10, - token_expiration: 5, - filter_expiration: 1, - backup: 1, federator_incoming: 5, federator_outgoing: 5, ingestion_queue: 50, web_push: 50, - mailer: 10, transmogrifier: 20, - scheduled_activities: 10, - poll_notifications: 10, background: 5, - remote_fetcher: 2, - attachments_cleanup: 1, - new_users_digest: 1, - mute_expire: 5, - search_indexing: 10, - rich_media_expiration: 2 + search_indexing: [limit: 10, paused: true], + slow: 1 ], plugins: [Oban.Plugins.Pruner], crontab: [ @@ -818,22 +828,27 @@ config :pleroma, :connections_pool, config :pleroma, :pools, federation: [ - size: 50, - max_waiting: 10, + size: 75, + max_waiting: 20, recv_timeout: 10_000 ], media: [ - size: 50, + size: 75, + max_waiting: 20, + recv_timeout: 15_000 + ], + rich_media: [ + size: 25, max_waiting: 20, recv_timeout: 15_000 ], upload: [ size: 25, - max_waiting: 5, + max_waiting: 20, recv_timeout: 15_000 ], default: [ - size: 10, + size: 50, max_waiting: 2, recv_timeout: 5_000 ] @@ -847,6 +862,10 @@ config :pleroma, :hackney_pools, max_connections: 50, timeout: 150_000 ], + rich_media: [ + max_connections: 50, + timeout: 150_000 + ], upload: [ max_connections: 25, timeout: 300_000 @@ -892,8 +911,6 @@ config :pleroma, Pleroma.User.Backup, process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ - {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] @@ -915,6 +932,19 @@ config :pleroma, Pleroma.Application, config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000 +config :pleroma, Pleroma.Search.QdrantSearch, + qdrant_url: "http://127.0.0.1:6333/", + qdrant_api_key: "", + openai_url: "http://127.0.0.1:11345", + # The healthcheck url has to be set to nil when used with the real openai + # API, as it doesn't have a healthcheck endpoint. + openai_healthcheck_url: "http://127.0.0.1:11345/health", + openai_model: "snowflake/snowflake-arctic-embed-xs", + openai_api_key: "", + qdrant_index_configuration: %{ + vectors: %{size: 384, distance: "Cosine"} + } + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs"diff --git a/config/description.exs b/config/description.exs@@ -138,6 +138,31 @@ config :pleroma, :config_description, [ }, %{ group: :pleroma, + key: Pleroma.Uploaders.IPFS, + type: :group, + description: "IPFS uploader-related settings", + children: [ + %{ + key: :get_gateway_url, + type: :string, + description: "GET Gateway URL", + suggestions: [ + "https://ipfs.mydomain.com/{CID}", + "https://{CID}.ipfs.mydomain.com/" + ] + }, + %{ + key: :post_gateway_url, + type: :string, + description: "POST Gateway URL", + suggestions: [ + "http://localhost:5001/" + ] + } + ] + }, + %{ + group: :pleroma, key: Pleroma.Uploaders.S3, type: :group, description: "S3 uploader-related settings", @@ -750,6 +775,18 @@ config :pleroma, :config_description, [ ] }, %{ + key: :rejected_instances, + type: {:list, :tuple}, + key_placeholder: "instance", + value_placeholder: "reason", + description: + "List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled", + suggestions: [ + {"rejected.com", "Reason"}, + {"*.rejected.com", "Reason"} + ] + }, + %{ key: :static_dir, type: :string, description: "Instance static directory", @@ -1792,6 +1829,12 @@ config :pleroma, :config_description, [ description: "Require HTTP signatures for AP fetches" }, %{ + key: :authorized_fetch_mode_exceptions, + type: {:list, :string}, + description: + "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)" + }, + %{ key: :note_replies_output_limit, type: :integer, description:diff --git a/config/dev.exs b/config/dev.exs@@ -35,8 +35,8 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # configured to run both http and https servers on # different ports. -# Do not include metadata nor timestamps in development logs -config :logger, :console, format: "[$level] $message\n" +# Do not include timestamps in development logs +config :logger, :console, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive.diff --git a/config/test.exs b/config/test.exs@@ -153,6 +153,12 @@ config :pleroma, Pleroma.Uploaders.S3, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock + +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, + http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock peer_module = if String.to_integer(System.otp_release()) >= 25 dodiff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md@@ -41,6 +41,7 @@ To add configuration to your config file, you can copy it from the base config. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. * `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send. +* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled. * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with older software for theses nicknames. @@ -284,6 +285,7 @@ Notes: * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches +* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this) ## Pleroma.User @@ -472,6 +474,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent. * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. +* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation. ### Pleroma.Web.Plugs.RemoteIp @@ -661,6 +664,19 @@ config :ex_aws, :s3, host: "s3.eu-central-1.amazonaws.com" ``` +#### Pleroma.Uploaders.IPFS + +* `post_gateway_url`: URL with port of POST Gateway (unauthenticated) +* `get_gateway_url`: URL of public GET Gateway + +Example: + +```elixir +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: "http://localhost:5001", + get_gateway_url: "http://{CID}.ipfs.mydomain.com" +``` + ### Upload filters #### Pleroma.Upload.Filter.AnonymizeFilenamediff --git a/docs/configuration/search.md b/docs/configuration/search.md@@ -10,6 +10,30 @@ To use built-in search that has no external dependencies, set the search module While it has no external dependencies, it has problems with performance and relevancy. +## QdrantSearch + +This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. + +The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`: + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch + +Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions. + +You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities. + +### Indexing and model options + +To see the available configuration options, check out the QdrantSearch section in `config/config.exs`. + +The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail. + +Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. + +E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024. + +When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other. + ## Meilisearch Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 milliondiff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md@@ -1751,3 +1751,53 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns ```json {} ``` + + +## `GET /api/v1/pleroma/admin/rules` + +### List rules + +- Response: JSON, list of rules + +```json +[ + { + "id": "1", + "priority": 1, + "text": "There are no rules", + "hint": null + } +] +``` + +## `POST /api/v1/pleroma/admin/rules` + +### Create a rule + +- Params: + - `text`: string, required, rule content + - `hint`: string, optional, rule description + - `priority`: integer, optional, rule ordering priority + +- Response: JSON, a single rule + +## `PATCH /api/v1/pleroma/admin/rules/:id` + +### Update a rule + +- Params: + - `text`: string, optional, rule content + - `hint`: string, optional, rule description + - `priority`: integer, optional, rule ordering priority + +- Response: JSON, a single rule + +## `DELETE /api/v1/pleroma/admin/rules/:id` + +### Delete a rule + +- Response: JSON, empty object + +```json +{} +```diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md@@ -295,9 +295,7 @@ See [Admin-API](admin_api.md) "id": "9umDrYheeY451cQnEe", "name": "Read later", "emoji": "🕓", - "source": { - "emoji": "🕓" - } + "emoji_url": null } ] ```diff --git a/docs/development/API/prometheus.md b/docs/development/API/prometheus.md@@ -1,44 +1,47 @@ -# Prometheus Metrics +# Prometheus / OpenTelemetry Metrics -Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. +Pleroma includes support for exporting metrics via the [prom_ex](https://github.com/akoutmos/prom_ex) library. +The metrics are exposed by a dedicated webserver/port to improve privacy and security. Config example: ``` -config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, - enabled: true, - auth: {:basic, "myusername", "mypassword"}, - ip_whitelist: ["127.0.0.1"], - path: "/api/pleroma/app_metrics", - format: :text -``` - -* `enabled` (Pleroma extension) enables the endpoint -* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs -* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation) -* `format` sets the output format (`:text` or `:protobuf`) -* `path` sets the path to app metrics page - - -## `/api/pleroma/app_metrics` +config :pleroma, Pleroma.PromEx, + disabled: false, + manual_metrics_start_delay: :no_delay, + drop_metrics_groups: [], + grafana: [ + host: System.get_env("GRAFANA_HOST", "http://localhost:3000"), + auth_token: System.get_env("GRAFANA_TOKEN"), + upload_dashboards_on_start: false, + folder_name: "BEAM", + annotate_app_lifecycle: true + ], + metrics_server: [ + port: 4021, + path: "/metrics", + protocol: :http, + pool_size: 5, + cowboy_opts: [], + auth_strategy: :none + ], + datasource: "Prometheus" -### Exports Prometheus application metrics - -* Method: `GET` -* Authentication: not required by default (see configuration options above) -* Params: none -* Response: text +``` -## Grafana +PromEx supports the ability to automatically publish dashboards to your Grafana server as well as register Annotations. If you do not wish to configure this capability you must generate the dashboard JSON files and import them directly. You can find the mix commands in the upstream [documentation](https://hexdocs.pm/prom_ex/Mix.Tasks.PromEx.Dashboard.Export.html). You can find the list of modules enabled in Pleroma for which you should generate dashboards for by examining the contents of the `lib/pleroma/prom_ex.ex` module. -### Config example +## prometheus.yml -The following is a config example to use with [Grafana](https://grafana.com) +The following is a bare minimum config example to use with [Prometheus](https://prometheus.io) or Prometheus-compatible software like [VictoriaMetrics](https://victoriametrics.com). ``` - - job_name: 'beam' - metrics_path: /api/pleroma/app_metrics - scheme: https +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'pleroma' + scheme: http static_configs: - - targets: ['pleroma.soykaf.com'] + - targets: ['pleroma.soykaf.com:4021'] ```diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md@@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 11.0以上 (同上) -- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git`diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include@@ -1,7 +1,7 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.11.0 <1.15 +* Elixir >=1.13.0 <1.15 * Erlang OTP >=22.2.0 (supported: <27) * git * file / libmagicdiff --git a/installation/nsfw-api.service b/installation/nsfw-api.service@@ -0,0 +1,15 @@ +[Unit] +Description=NSFW API +After=docker.service +Requires=docker.service + +[Service] +TimeoutStartSec=0 +Restart=always +ExecStartPre=-/usr/bin/docker stop %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest +ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest + +[Install] +WantedBy=multi-user.targetdiff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex@@ -0,0 +1,80 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["create_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end + end + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + limit: :integer + ] + ) + + start_pleroma() + + limit = Keyword.get(options, :limit, 100_000) + + per_step = 1000 + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(100) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +enddiff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex@@ -14,6 +14,7 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @compile_env Mix.env() def name, do: @name def version, do: @version @@ -51,7 +52,11 @@ defmodule Pleroma.Application do Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + + if @compile_env != :test do + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + end + Pleroma.ApplicationRequirements.verify!() load_custom_modules() Pleroma.Docs.JSON.compile() @@ -109,7 +114,8 @@ defmodule Pleroma.Application do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -162,7 +168,8 @@ defmodule Pleroma.Application do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] enddiff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex@@ -28,6 +28,7 @@ defmodule Pleroma.ApplicationRequirements do |> check_welcome_message_config!() |> check_rum!() |> check_repo_pool_size!() + |> check_mrfs() |> handle_result() end @@ -234,4 +235,25 @@ defmodule Pleroma.ApplicationRequirements do true end end + + defp check_mrfs(:ok) do + mrfs = Config.get!([:mrf, :policies]) + + missing_mrfs = + Enum.reduce(mrfs, [], fn x, acc -> + if Code.ensure_compiled(x) do + acc + else + acc ++ [x] + end + end) + + if Enum.empty?(missing_mrfs) do + :ok + else + {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"} + end + end + + defp check_mrfs(result), do: result enddiff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do "context_id", "deleted_activity_id", "pleroma_internal", - "generator" + "generator", + "rules" ] )diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex@@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do def parse_address(ip) do :inet.parse_address(ip) end + + def parse_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse_cidr!(proxy, true) + end enddiff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex@@ -0,0 +1,4 @@ +defmodule Pleroma.HTTPSignaturesAPI do + @callback validate_conn(conn :: Plug.Conn.t()) :: boolean + @callback signature_for_conn(conn :: Plug.Conn.t()) :: map +enddiff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex@@ -73,6 +73,7 @@ defmodule Pleroma.Notification do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -280,15 +281,10 @@ defmodule Pleroma.Notification do select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - - for_user_query(user) - |> where([n], n.id in ^notification_ids) - |> Repo.all() + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_last_read_id(user, "notifications") + |> Repo.transaction() end @spec read_one(User.t(), String.t()) :: @@ -299,10 +295,6 @@ defmodule Pleroma.Notification do |> Multi.update(:update, changeset(notification, %{seen: true})) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end end end @@ -361,37 +353,38 @@ defmodule Pleroma.Notification do end end - @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} - def create_notifications(activity, options \\ []) + @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []} + def create_notifications(activity) - def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do + def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity, fetch: false) if object && object.data["type"] == "Answer" do {:ok, []} else - do_create_notifications(activity, options) + do_create_notifications(activity) end end - def create_notifications(%Activity{data: %{"type" => type}} = activity, options) + def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do - do_create_notifications(activity, options) + do_create_notifications(activity) end - def create_notifications(_, _), do: {:ok, []} + def create_notifications(_), do: {:ok, []} - defp do_create_notifications(%Activity{} = activity, options) do - do_send = Keyword.get(options, :do_send, true) + defp do_create_notifications(%Activity{} = activity) do + enabled_receivers = get_notified_from_activity(activity) - {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) - potential_receivers = enabled_receivers ++ disabled_receivers + enabled_subscribers = get_notified_subscribers_from_activity(activity) notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) - end) + (Enum.map(enabled_receivers, fn user -> + create_notification(activity, user) + end) ++ + Enum.map(enabled_subscribers -- enabled_receivers, fn user -> + create_notification(activity, user, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -450,7 +443,6 @@ defmodule Pleroma.Notification do # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do - do_send = Keyword.get(opts, :do_send, true) type = Keyword.get(opts, :type, type_from_activity(activity)) unless skip?(activity, user, opts) do @@ -465,11 +457,6 @@ defmodule Pleroma.Notification do |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - if do_send do - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end - notification end end @@ -527,13 +514,28 @@ defmodule Pleroma.Notification do |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) - notification_enabled_users = - Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + end + + def get_notified_from_activity(_, _local_only), do: [] + + def get_notified_subscribers_from_activity(activity, local_only \\ true) - {notification_enabled_users, potential_receivers -- notification_enabled_users} + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_subscribers_from_activity(_, _), do: [] # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) @@ -576,7 +578,6 @@ defmodule Pleroma.Notification do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end @@ -643,6 +644,7 @@ defmodule Pleroma.Notification do def skip?(%Activity{} = activity, %User{} = user, opts) do [ :self, + :internal, :invisible, :block_from_strangers, :recently_followed, @@ -662,6 +664,12 @@ defmodule Pleroma.Notification do end end + def skip?(:internal, %Activity{} = activity, _user, _opts) do + actor = activity.data["actor"] + user = User.get_cached_by_ap_id(actor) + User.internal?(user) + end + def skip?(:invisible, %Activity{} = activity, _user, _opts) do actor = activity.data["actor"] user = User.get_cached_by_ap_id(actor) @@ -748,4 +756,12 @@ defmodule Pleroma.Notification do ) |> Repo.update_all(set: [seen: true]) end + + @spec send(list(Notification.t())) :: :ok + def send(notifications) do + Enum.each(notifications, fn notification -> + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end) + end enddiff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex@@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Rule do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.Rule + + schema "rules" do + field(:priority, :integer, default: 0) + field(:text, :string) + field(:hint, :string) + + timestamps() + end + + def changeset(%Rule{} = rule, params \\ %{}) do + rule + |> cast(params, [:priority, :text, :hint]) + |> validate_required([:text]) + end + + def query do + Rule + |> order_by(asc: :priority) + |> order_by(asc: :id) + end + + def get(ids) when is_list(ids) do + from(r in __MODULE__, where: r.id in ^ids) + |> Repo.all() + end + + def get(id), do: Repo.get(__MODULE__, id) + + def exists?(id) do + from(r in __MODULE__, where: r.id == ^id) + |> Repo.exists?() + end + + def create(params) do + {:ok, rule} = + %Rule{} + |> changeset(params) + |> Repo.insert() + + rule + end + + def update(params, id) do + {:ok, rule} = + get(id) + |> changeset(params) + |> Repo.update() + + rule + end + + def delete(id) do + get(id) + |> Repo.delete() + end +enddiff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex@@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do def job_query(scheduled_activity_id) do from(j in Oban.Job, - where: j.queue == "scheduled_activities", + where: j.queue == "federator_outgoing", where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) ) enddiff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex@@ -10,8 +10,12 @@ defmodule Pleroma.Search do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module.healthcheck_endpoints + end enddiff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex@@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def remove_from_index(_object), do: :ok + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) enddiff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex@@ -0,0 +1,86 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if check(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} + end + + def state, do: GenServer.call(__MODULE__, :state) + + def check([]), do: true + + def check(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + def check(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +enddiff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex@@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + defp meili_headers do private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -178,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end enddiff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex@@ -0,0 +1,182 @@ +defmodule Pleroma.Search.QdrantSearch do + @behaviour Pleroma.Search.SearchBackend + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + alias __MODULE__.OpenAIClient + alias __MODULE__.QdrantClient + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] + + @impl true + def create_index do + payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + + with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def drop_index do + with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do + :ok + else + e -> {:error, e} + end + end + + def get_embedding(text) do + with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <- + OpenAIClient.post("/v1/embeddings", %{ + input: text, + model: Config.get([Pleroma.Search.QdrantSearch, :openai_model]) + }) do + {:ok, embedding} + else + _ -> + {:error, "Failed to get embedding"} + end + end + + defp actor_from_activity(%{data: %{"actor" => actor}}) do + actor + end + + defp actor_from_activity(_), do: nil + + defp build_index_payload(activity, embedding) do + actor = actor_from_activity(activity) + published_at = activity.data["published"] + + %{ + points: [ + %{ + id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), + vector: embedding, + payload: %{actor: actor, published_at: published_at} + } + ] + } + end + + defp build_search_payload(embedding, options) do + base = %{ + vector: embedding, + limit: options[:limit] || 20, + offset: options[:offset] || 0 + } + + if author = options[:author] do + Map.put(base, :filter, %{ + must: [%{key: "actor", match: %{value: author.ap_id}}] + }) + else + base + end + end + + @impl true + def add_to_index(activity) do + # This will only index public or unlisted notes + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + with {:ok, embedding} <- get_embedding(maybe_search_data.content), + {:ok, %{status: 200}} <- + QdrantClient.put( + "/collections/posts/points", + build_index_payload(activity, embedding) + ) do + :ok + else + e -> {:error, e} + end + else + :ok + end + end + + @impl true + def remove_from_index(object) do + activity = Activity.get_by_object_ap_id_with_object(object.data["id"]) + id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!() + + with {:ok, %{status: 200}} <- + QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def search(user, original_query, options) do + query = "Represent this sentence for searching relevant passages: #{original_query}" + + with {:ok, embedding} <- get_embedding(query), + {:ok, %{body: %{"result" => result}}} <- + QdrantClient.post( + "/collections/posts/points/search", + build_search_payload(embedding, options) + ) do + ids = + Enum.map(result, fn %{"id" => id} -> + Ecto.UUID.dump!(id) + end) + + from(a in Activity, where: a.id in ^ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) + |> Pleroma.Repo.all() + |> maybe_fetch(user, original_query) + else + _ -> + [] + end + end + + @impl true + def healthcheck_endpoints do + qdrant_health = + Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]) + |> URI.parse() + |> Map.put(:path, "/healthz") + |> URI.to_string() + + openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url]) + + [qdrant_health, openai_health] |> Enum.filter(& &1) + end +end + +defmodule Pleroma.Search.QdrantSearch.OpenAIClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"Authorization", + "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]) +end + +defmodule Pleroma.Search.QdrantSearch.QdrantClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]) +enddiff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex@@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Create the index + """ + @callback create_index() :: :ok | {:error, any()} + + @doc """ + Drop the index + """ + @callback drop_index() :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil enddiff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex@@ -44,8 +44,7 @@ defmodule Pleroma.Signature do defp remove_suffix(uri, []), do: uri def fetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -55,8 +54,7 @@ defmodule Pleroma.Signature do end def refetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} @@ -66,6 +64,16 @@ defmodule Pleroma.Signature do end end + def get_actor_id(conn) do + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + {:ok, actor_id} <- key_id_to_actor_id(kid) do + {:ok, actor_id} + else + e -> + {:error, e} + end + end + def sign(%User{keys: keys} = user, headers) do with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex@@ -239,8 +239,12 @@ defmodule Pleroma.Upload do "" end - [base_url, path] - |> Path.join() + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -277,6 +281,9 @@ defmodule Pleroma.Upload do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" enddiff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex@@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Tesla.Multipart + + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + defp get_final_url(method) do + config = @config_impl.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + + def put_file_endpoint do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash key"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway upload failed"} + end + end + + @impl true + def delete_file(file) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do + {:ok, %{status: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +enddiff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex@@ -1404,6 +1404,40 @@ defmodule Pleroma.User do |> Repo.all() end + @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do + friends = + get_friends_query(current_user) + |> where([u], not u.hide_follows) + |> select([u], u.id) + + User.Query.build(%{is_active: true}) + |> where([u], u.id not in ^[user.id, current_user.id]) + |> join(:inner, [u], r in FollowingRelationship, + as: :followers_relationships, + on: r.following_id == ^user.id and r.follower_id == u.id + ) + |> where([followers_relationships: r], r.state == ^:follow_accept) + |> where([followers_relationships: r], r.follower_id in subquery(friends)) + end + + def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do + user + |> get_familiar_followers_query(current_user, nil) + |> User.Query.paginate(page, 20) + end + + @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user), + do: get_familiar_followers_query(user, current_user, nil) + + @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())} + def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do + user + |> get_familiar_followers_query(current_user, page) + |> Repo.all() + end + def increase_note_count(%User{} = user) do User |> where(id: ^user.id)diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex@@ -200,7 +200,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def notify_and_stream(activity) do - Notification.create_notifications(activity) + {:ok, notifications} = Notification.create_notifications(activity) + Notification.send(notifications) original_activity = case activity do @@ -978,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_replies(query, %{exclude_replies: true}) do from( - [_activity, object] in query, - where: fragment("?->>'inReplyTo' is null", object.data) + [activity, object] in query, + where: + fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data) ) end @@ -1259,6 +1261,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_quote_url(query, _), do: query + defp restrict_rule(query, %{rule_id: rule_id}) do + from( + activity in query, + where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id) + ) + end + + defp restrict_rule(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1421,6 +1432,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_rule(opts) |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts)diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex@@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -521,6 +522,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn end + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload(diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex@@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.Config + alias Pleroma.User + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp user_has_posted?(%User{} = u), do: u.note_count > 0 + + defp user_has_age?(%User{} = u) do + user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond) + diff >= user_age_limit + end + + defp good_reputation?(%User{} = u) do + user_has_age?(u) and user_has_posted?(u) + end + + # copied from HellthreadPolicy + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + follower_collection = + User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + + if Enum.member?(recipients, Pleroma.Constants.as_public()) do + recipients = + recipients + |> List.delete(Pleroma.Constants.as_public()) + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients = + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + defp object_has_recipients?(%{"object" => object} = activity) do + {_, object_count} = get_recipient_count(object) + {_, activity_count} = get_recipient_count(activity) + object_count + activity_count > 0 + end + + defp object_has_recipients?(object) do + {_, count} = get_recipient_count(object) + count > 0 + end + + @impl true + def filter(%{"type" => "Create", "actor" => actor} = activity) do + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), + {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)}, + {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do + {:ok, activity} + else + {:ok, %User{local: true}} -> + {:ok, activity} + + {:has_mentions, false} -> + {:ok, activity} + + {:good_reputation, false} -> + {:reject, "[AntiMentionSpamPolicy] User rejected"} + + {:error, _} -> + {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"} + + e -> + {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +enddiff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex@@ -0,0 +1,142 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do + @moduledoc """ + Dynamic activity filtering based on an RBL database + + This MRF makes queries to a custom DNS server which will + respond with values indicating the classification of the domain + the activity originated from. This method has been widely used + in the email anti-spam industry for very fast reputation checks. + + e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK + Other values such as 127.0.0.2 may be used for specific classifications. + + Information for why the host is blocked can be stored in a corresponding TXT record. + + This method is fail-open so if the queries fail the activites are accepted. + + An example of software meant for this purpsoe is rbldnsd which can be found + at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at + https://git.pleroma.social/feld/rbldnsd + + It is highly recommended that you run your own copy of rbldnsd and use an + external mechanism to sync/share the contents of the zone file. This is + important to keep the latency on the queries as low as possible and prevent + your DNS server from being attacked so it fails and content is permitted. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + require Logger + + @query_retries 1 + @query_timeout 500 + + @impl true + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_rbl(actor_info, object) do + {:ok, object} + else + _ -> {:reject, "[DNSRBLPolicy]"} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe do + mrf_dnsrbl = + Config.get(:mrf_dnsrbl) + |> Enum.into(%{}) + + {:ok, %{mrf_dnsrbl: mrf_dnsrbl}} + end + + @impl true + def config_description do + %{ + key: :mrf_dnsrbl, + related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy", + label: "MRF DNSRBL", + description: "DNS RealTime Blackhole Policy", + children: [ + %{ + key: :nameserver, + type: {:string}, + description: "DNSRBL Nameserver to Query (IP or hostame)", + suggestions: ["127.0.0.1"] + }, + %{ + key: :port, + type: {:string}, + description: "Nameserver port", + suggestions: ["53"] + }, + %{ + key: :zone, + type: {:string}, + description: "Root zone for querying", + suggestions: ["bl.pleroma.com"] + } + ] + } + end + + defp check_rbl(%{host: actor_host}, object) do + with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), + zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do + query = + Enum.join([actor_host, zone], ".") + |> String.to_charlist() + + rbl_response = rblquery(query) + + if Enum.empty?(rbl_response) do + {:ok, object} + else + Task.start(fn -> + reason = rblquery(query, :txt) || "undefined" + + Logger.warning( + "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}" + ) + end) + + :error + end + else + _ -> {:ok, object} + end + end + + defp get_rblhost_ip(rblhost) do + case rblhost |> String.to_charlist() |> :inet_parse.address() do + {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address() + _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()} + end + end + + defp rblquery(query, type \\ :a) do + config = Config.get([:mrf_dnsrbl]) + + case get_rblhost_ip(config[:nameserver]) do + {:ok, rblnsip} -> + :inet_res.lookup(query, :in, type, + nameservers: [{rblnsip, config[:port]}], + timeout: @query_timeout, + retry: @query_retries + ) + + _ -> + [] + end + end +enddiff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex@@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -27,17 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) - end + fetch(prefetch_url) end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fndiff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex@@ -0,0 +1,265 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api + + def build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> fix_path() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + def parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warn(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + def check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + end + + def check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + def check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + def check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + _ -> {:reject, "NSFW: Attachment rejected"} + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" + end + end + + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + def mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + + @impl true + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end +enddiff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType])diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex@@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Push alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker @@ -125,7 +124,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do nil end - {:ok, notifications} = Notification.create_notifications(object, do_send: false) + {:ok, notifications} = Notification.create_notifications(object) meta = meta @@ -184,7 +183,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do liked_object = Object.get_by_ap_id(object.data["object"]) Utils.add_like_to_object(object, liked_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -202,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, notifications} = Notification.create_notifications(activity) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object) @@ -256,11 +259,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Utils.add_announce_to_object(object, announced_object) - if !User.internal?(user) do - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) - ap_streamer().stream_out(object) - end + if !User.internal?(user), do: ap_streamer().stream_out(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -281,7 +286,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reacted_object = Object.get_by_ap_id(object.data["object"]) Utils.add_emoji_reaction_to_object(object, reacted_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -585,10 +594,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do defp send_notifications(meta) do Keyword.get(meta, :notifications, []) - |> Enum.each(fn notification -> - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end) + |> Notification.send() meta enddiff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex@@ -721,14 +721,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Flag-related helpers @spec make_flag_data(map(), map()) :: map() - def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do + def make_flag_data( + %{actor: actor, context: context, content: content} = params, + additional + ) do %{ "type" => "Flag", "actor" => actor.ap_id, "content" => content, "object" => build_flag_object(params), "context" => context, - "state" => "open" + "state" => "open", + "rules" => Map.get(params, :rules, nil) } |> Map.merge(additional) enddiff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex@@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Rule + alias Pleroma.Web.Plugs.OAuthScopesPlug + + import Pleroma.Web.ControllerHelper, + only: [ + json_response: 3 + ] + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:create, :update, :delete] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) + + action_fallback(AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation + + def index(conn, _) do + rules = + Rule.query() + |> Repo.all() + + render(conn, "index.json", rules: rules) + end + + def create(%{body_params: params} = conn, _) do + rule = + params + |> Rule.create() + + render(conn, "show.json", rule: rule) + end + + def update(%{body_params: params} = conn, %{id: id}) do + rule = + params + |> Rule.update(id) + + render(conn, "show.json", rule: rule) + end + + def delete(conn, %{id: id}) do + with {:ok, _} <- Rule.delete(id) do + json(conn, %{}) + else + _ -> json_response(conn, :bad_request, "") + end + end +enddiff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex@@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.RuleView alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView @@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do as: :activity }), state: report.data["state"], - notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) + notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}), + rules: rules(Map.get(report.data, "rules", nil)) } end @@ -71,4 +74,16 @@ defmodule Pleroma.Web.AdminAPI.ReportView do created_at: Utils.to_masto_date(inserted_at) } end + + defp rules(nil) do + [] + end + + defp rules(rule_ids) do + rules = + rule_ids + |> Rule.get() + + render(RuleView, "index.json", rules: rules) + end enddiff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex@@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleView do + use Pleroma.Web, :view + + require Pleroma.Constants + + def render("index.json", %{rules: rules} = _opts) do + render_many(rules, __MODULE__, "show.json") + end + + def render("show.json", %{rule: rule} = _opts) do + %{ + id: to_string(rule.id), + priority: rule.priority, + text: rule.text, + hint: rule.hint + } + end +enddiff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex@@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do "Frontend management", "Instance configuration", "Instance documents", + "Instance rule managment", "Invites", "MediaProxy cache", "OAuth application management",diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do alias Pleroma.Web.ApiSpec.Schemas.ActorType alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.List alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + def familiar_followers_operation do + %Operation{ + tags: ["Retrieve account information"], + summary: "Followers that you follow", + operationId: "AccountController.familiar_followers", + description: + "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.", + security: [%{"oAuth" => ["read:follows"]}], + parameters: [ + Operation.parameter( + :id, + :query, + %Schema{ + oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}] + }, + "Account IDs", + example: "123" + ) + ], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: %Schema{ + title: "Account", + type: :object, + properties: %{ + id: FlakeID, + accounts: %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: Account, + example: [Account.schema().example] + } + } + } + }) + } + } + end + defp create_request do %Schema{ title: "AccountCreateRequest",diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex@@ -31,6 +31,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do "Filter by report state" ), Operation.parameter( + :rule_id, + :query, + %Schema{type: :string}, + "Filter by selected rule id" + ), + Operation.parameter( :limit, :query, %Schema{type: :integer}, @@ -169,6 +175,17 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do inserted_at: %Schema{type: :string, format: :"date-time"} } } + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } } } }diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex@@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Retrieve list of instance rules", + operationId: "AdminAPI.RuleController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: rule() + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Create new rule", + operationId: "AdminAPI.RuleController.create", + security: [%{"oAuth" => ["admin:write"]}], + parameters: admin_api_params(), + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Modify existing rule", + operationId: "AdminAPI.RuleController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Delete rule", + operationId: "AdminAPI.RuleController.delete", + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + security: [%{"oAuth" => ["admin:write"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + type: :object, + required: [:text], + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp update_request do + %Schema{ + type: :object, + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp rule do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } + end +enddiff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex@@ -46,10 +46,30 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def rules_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve list of instance rules", + operationId: "InstanceController.rules", + responses: %{ + 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + } + } + end + defp instance do %Schema{ type: :object, properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + } + } + }, uri: %Schema{type: :string, description: "The domain name of the instance"}, title: %Schema{type: :string, description: "The title of the website"}, description: %Schema{ @@ -172,7 +192,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do "urls" => %{ "streaming_api" => "wss://lain.com" }, - "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)" + "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)", + "rules" => array_of_rules() } } end @@ -272,6 +293,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "Instance configuration", properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + }, + max_pinned_statuses: %Schema{ + type: :integer, + description: "The maximum number of pinned statuses for each account." + } + } + }, urls: %Schema{ type: :object, properties: %{ @@ -285,6 +319,11 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "A map with poll limits for local statuses", properties: %{ + characters_reserved_per_url: %Schema{ + type: :integer, + description: + "Each URL in a status will be assumed to be exactly this many characters." + }, max_characters: %Schema{ type: :integer, description: "Posts character limit (CW/Subject included in the counter)" @@ -344,4 +383,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do example: ["pleroma.site", "lain.com", "bikeshed.party"] } end + + defp array_of_rules do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + } + end enddiff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex@@ -202,7 +202,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status" ], description: """ The type of event that resulted in the notification. @@ -216,6 +217,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status """ } enddiff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex@@ -5,7 +5,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do Operation.response( "A Notification or array of Notifications", "application/json", - %Schema{ - anyOf: [ - %Schema{type: :array, items: NotificationOperation.notification()}, - NotificationOperation.notification() - ] - } + %Schema{type: :string} ), 400 => Operation.response("Bad Request", "application/json", ApiError) }diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex@@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do default: false, description: "If the account is remote, should the report be forwarded to the remote admin?" + }, + rule_ids: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: "Array of rules" } }, required: [:account_id], @@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do "account_id" => "123", "status_ids" => ["1337"], "comment" => "bad status!", - "forward" => "false" + "forward" => "false", + "rule_ids" => ["3"] } } enddiff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex@@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } },diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex@@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Formatter alias Pleroma.ModerationLog alias Pleroma.Object + alias Pleroma.Rule alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.UserRelationship @@ -568,14 +569,16 @@ defmodule Pleroma.Web.CommonAPI do def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), - {:ok, statuses} <- get_report_statuses(account, data) do + {:ok, statuses} <- get_report_statuses(account, data), + rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do ActivityPub.flag(%{ context: Utils.generate_context_id(), actor: user, account: account, statuses: statuses, content: content_html, - forward: Map.get(data, :forward, false) + forward: Map.get(data, :forward, false), + rules: rules }) end end @@ -587,6 +590,15 @@ defmodule Pleroma.Web.CommonAPI do end end + defp get_report_rules(nil) do + nil + end + + defp get_report_rules(rule_ids) do + rule_ids + |> Enum.filter(&Rule.exists?/1) + end + def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids}diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex@@ -129,8 +129,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)} + defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do + add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) + end + + defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do + activity = Activity.get_by_id(id) + + params = + if is_nil(activity) do + # Deleted activities are returned as nil + Map.put(params, :in_reply_to_status_id, :deleted) + else + Map.put(params, :in_reply_to_status_id, activity) + end + + in_reply_to(%{draft | params: params}) end defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) dodiff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex@@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug)diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex@@ -44,7 +44,7 @@ defmodule Pleroma.Web.Federator do end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) end def incoming_ap_doc(params) dodiff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex@@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock] ) - plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships) + plug( + OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers] + ) plug( OAuthScopesPlug, @@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do ) end + @doc "GET /api/v1/accounts/familiar_followers" + def familiar_followers( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _id + ) do + users = + User.get_all_by_ids(List.wrap(id)) + |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)}) + + conn + |> render("familiar_followers.json", + for: user, + users: users, + as: :user + ) + end + + defp get_familiar_followers(%{id: id} = user, %{id: id}) do + User.get_familiar_followers(user, user) + end + + defp get_familiar_followers(%{hide_followers: true}, _current_user) do + [] + end + + defp get_familiar_followers(user, current_user) do + User.get_familiar_followers(user, current_user) + end + @doc "GET /api/v1/identity_proofs" def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) enddiff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex@@ -25,4 +25,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) end + + @doc "GET /api/v1/instance/rules" + def rules(conn, _params) do + render(conn, "rules.json") + end enddiff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex@@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do pleroma:emoji_reaction poll update + status } # GET /api/v1/notificationsdiff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex@@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("familiar_followers.json", %{users: users} = opts) do + opts = + opts + |> Map.merge(%{as: :user}) + |> Map.delete(:users) + + users + |> render_many(AccountView, "familiar_followers.json", opts) + end + + def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do + accounts = + accounts + |> render_many(AccountView, "show.json", opts) + |> Enum.filter(&Enum.any?/1) + + %{id: id, accounts: accounts} + end + defp do_render("show.json", %{user: user} = opts) do self = opts[:for] == userdiff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex@@ -76,12 +76,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) end + def render("rules.json", _) do + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> render_many(__MODULE__, "rule.json", as: :rule) + end + + def render("rule.json", %{rule: rule}) do + %{ + id: to_string(rule.id), + text: rule.text, + hint: rule.hint || "" + } + end + defp common_information(instance) do %{ - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", languages: Keyword.get(instance, :languages, ["en"]), - rules: [] + rules: render(__MODULE__, "rules.json"), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" } end @@ -138,6 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) + rejected = Config.get([:instance, :rejected_instances], []) if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() @@ -157,6 +172,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Map.new() }) + |> Map.put( + :rejected_instances, + rejected + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() + ) else %{} end @@ -213,6 +234,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do defp configuration2 do configuration() + |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0)) + |> put_in([:statuses, :characters_reserved_per_url], 0) |> Map.merge(%{ urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(),diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex@@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts)diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex@@ -624,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -631,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta)diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex@@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length)diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex@@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do } = conn, _ ) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - render(conn, "show.json", notification: notification, for: user) + with {:ok, _} <- Notification.read_one(user, notification_id) do + conn + |> json("ok") else {:error, message} -> conn @@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do conn, _ ) do - notifications = - user - |> Notification.set_read_up_to(max_id) - |> Enum.take(80) - - render(conn, "index.json", notifications: notifications, for: user) + with {:ok, _} <- Notification.set_read_up_to(user, max_id) do + conn + |> json("ok") + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end end enddiff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex@@ -3,26 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do - alias Pleroma.Config import Plug.Conn require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(opts), do: opts def call(conn, _options) do - if Config.get([:http_security, :enabled]) do + if @config_impl.get([:http_security, :enabled]) do conn |> merge_resp_headers(headers()) - |> maybe_send_sts_header(Config.get([:http_security, :sts])) + |> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) else conn end end def primary_frontend do - with %{"name" => frontend} <- Config.get([:frontends, :primary]), - available <- Config.get([:frontends, :available]), + with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), + available <- @config_impl.get([:frontends, :available]), %{} = primary_frontend <- Map.get(available, frontend) do {:ok, primary_frontend} end @@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def headers do - referrer_policy = Config.get([:http_security, :referrer_policy]) - report_uri = Config.get([:http_security, :report_uri]) + referrer_policy = @config_impl.get([:http_security, :referrer_policy]) + report_uri = @config_impl.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() headers = [ @@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do @csp_start [Enum.join(static_csp_rules, ";") <> ";"] defp csp_string do - scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() - report_uri = Config.get([:http_security, :report_uri]) + report_uri = @config_impl.get([:http_security, :report_uri]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = - if Config.get([:media_proxy, :enabled]) && - !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do + if @config_impl.get([:media_proxy, :enabled]) && + !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() { @@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end connect_src = - if Config.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + if @config_impl.get([:http_security, :allow_unsafe_eval]) do + if @config_impl.get([:env]) == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self' 'wasm-unsafe-eval'" + end else - "script-src 'self' 'wasm-unsafe-eval'" + "script-src 'self'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do defp build_csp_multimedia_source_list do media_proxy_whitelist = [:media_proxy, :whitelist] - |> Config.get() + |> @config_impl.get() |> build_csp_from_whitelist([]) - captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = Config.get([captcha_method, :endpoint]) + captcha_method = @config_impl.get([Pleroma.Captcha, :method]) + captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) base_endpoints = [ @@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do [Pleroma.Upload, :base_url], [Pleroma.Uploaders.S3, :public_endpoint] ] - |> Enum.map(&Config.get/1) + |> Enum.map(&@config_impl.get/1) [captcha_endpoint | base_endpoints] |> Enum.map(&build_csp_param/1) @@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def warn_if_disabled do - unless Config.get([:http_security, :enabled]) do + unless Pleroma.Config.get([:http_security, :enabled]) do Logger.warning(" .i;;;;i. iYcviii;vXY: @@ -245,8 +250,8 @@ your instance and your users via malicious posts: end defp maybe_send_sts_header(conn, true) do - max_age_sts = Config.get([:http_security, :sts_max_age]) - max_age_ct = Config.get([:http_security, :ct_max_age]) + max_age_sts = @config_impl.get([:http_security, :sts_max_age]) + max_age_ct = @config_impl.get([:http_security, :ct_max_age]) merge_resp_headers(conn, [ {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex@@ -3,10 +3,22 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do + alias Pleroma.Helpers.InetHelper + import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] + + alias Pleroma.Web.ActivityPub.MRF + require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @http_signatures_impl Application.compile_env( + :pleroma, + [__MODULE__, :http_signatures_impl], + HTTPSignatures + ) + def init(options) do options end @@ -19,7 +31,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do if get_format(conn) in ["json", "activity+json"] do conn |> maybe_assign_valid_signature() + |> maybe_assign_actor_id() |> maybe_require_signature() + |> maybe_filter_requests() else conn end @@ -33,7 +47,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do |> put_req_header("(request-target)", request_target) |> put_req_header("@request-target", request_target) - HTTPSignatures.validate_conn(conn) + @http_signatures_impl.validate_conn(conn) end defp validate_signature(conn) do @@ -83,20 +97,63 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do + adapter = Application.get_env(:http_signatures, :adapter) + + {:ok, actor_id} = adapter.get_actor_id(conn) + + assign(conn, :actor_id, actor_id) + end + + defp maybe_assign_actor_id(conn), do: conn + defp has_signature_header?(conn) do conn |> get_req_header("signature") |> Enum.at(0, false) end defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn - defp maybe_require_signature(conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do + exceptions = + @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], []) + |> Enum.map(&InetHelper.parse_cidr/1) + + if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do + conn + else + conn + |> put_status(:unauthorized) + |> text("Request not signed") + |> halt() + end + else conn - |> put_status(:unauthorized) - |> text("Request not signed") - |> halt() + end + end + + defp maybe_filter_requests(%{halted: true} = conn), do: conn + + defp maybe_filter_requests(conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and + conn.assigns[:actor_id] do + %{host: host} = URI.parse(conn.assigns.actor_id) + + if MRF.subdomain_match?(rejected_domains(), host) do + conn + |> put_status(:unauthorized) + |> halt() + else + conn + end else conn end end + + defp rejected_domains do + @config_impl.get([:instance, :rejected_instances]) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + end enddiff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex@@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +enddiff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex@@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +enddiff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do """ alias Pleroma.Config + alias Pleroma.Helpers.InetHelper import Plug.Conn @behaviour Plug @@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do proxies = Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&maybe_add_cidr/1) + |> Enum.map(&InetHelper.parse_cidr/1) {headers, proxies} end - - defp maybe_add_cidr(proxy) when is_binary(proxy) do - proxy = - cond do - "/" in String.codepoints(proxy) -> proxy - InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" - InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" - end - - InetCidr.parse_cidr!(proxy, true) - end enddiff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex@@ -192,6 +192,7 @@ defmodule Pleroma.Web.Push.Impl do def format_title(%{type: type}, mastodon_type) do case mastodon_type || type do "mention" -> "New Mention" + "status" -> "New Status" "follow" -> "New Follower" "follow_request" -> "New Follow Request" "reblog" -> "New Repeat"diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex@@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp http_options do [ - pool: :media, + pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000) ] enddiff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex@@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do %URI{host: host, query: query} = URI.parse(image) is_binary(host) and String.contains?(host, "amazonaws.com") and - String.contains?(query, "X-Amz-Expires") + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nildiff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex@@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do @@ -292,6 +300,11 @@ defmodule Pleroma.Web.Router do post("/frontends/install", FrontendController, :install) post("/backups", AdminAPIController, :create_backup) + + get("/rules", RuleController, :index) + post("/rules", RuleController, :create) + patch("/rules/:id", RuleController, :update) + delete("/rules/:id", RuleController, :delete) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) @@ -633,6 +646,7 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", AccountController, :update_credentials) get("/accounts/relationships", AccountController, :relationships) + get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) @@ -764,6 +778,7 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/rules", InstanceController, :rules) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show)diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex@@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, request_url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} enddiff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex@@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + use Pleroma.Workers.WorkerHelper, queue: "slow" @impl Oban.Worker def perform(%Job{diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex@@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job alias Pleroma.User.Backupdiff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex@@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(_job) dodiff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex@@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) dodiff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex@@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) dodiff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" + use Pleroma.Workers.WorkerHelper, queue: "background" alias Pleroma.Activity alias Pleroma.Notificationdiff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do Worker which purges expired activity. """ - use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -59,7 +59,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", - where: j.queue == "activity_expiration", + where: j.queue == "slow", where: fragment("?->>'activity_id' = ?", j.args, ^id) ) |> Pleroma.Repo.one()diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do def get_expiration(id) do from(j in Job, where: j.state == "scheduled", - where: j.queue == "filter_expiration", + where: j.queue == "background", where: fragment("?->'filter_id' = ?", j.args, ^id) ) |> Repo.one()diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :token_expiration, max_attempts: 1 + use Oban.Worker, queue: :background, max_attempts: 1 @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex@@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) dodiff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do alias Pleroma.Web.RichMedia.Card use Oban.Worker, - queue: :rich_media_expiration + queue: :background @impl Oban.Worker def perform(%Job{args: %{"url" => url} = _args}) dodiff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex@@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" alias Pleroma.Repo alias Pleroma.ScheduledActivitydiff --git a/mix.exs b/mix.exs@@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do [ app: :pleroma, version: version("2.6.52"), - elixir: "~> 1.11", + elixir: "~> 1.13", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors()],diff --git a/priv/gettext/config_descriptions.pot b/priv/gettext/config_descriptions.pot@@ -5973,3 +5973,87 @@ msgstr "" msgctxt "config label at :pleroma-:instance > :languages" msgid "Languages" msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji" +msgid "Reject or force-unlisted emojis whose URLs or names match a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html)." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid " A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid " A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_shortcode" +msgid " A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_url" +msgid " A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "The number of activities to fetch in the backup job for each chunk." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji" +msgid "MRF Emoji" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid "Federated timeline removal shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid "Federated timeline removal url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_shortcode" +msgid "Remove shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_url" +msgid "Remove url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "Process Chunk Size" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "Process Wait Time" +msgstr ""diff --git a/priv/gettext/errors.pot b/priv/gettext/errors.pot@@ -110,7 +110,7 @@ msgstr "" msgid "Can't display this activity" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:334 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:346 #, elixir-autogen, elixir-format msgid "Can't find user" msgstr "" @@ -198,7 +198,7 @@ msgstr "" msgid "Invalid password." msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:279 #, elixir-autogen, elixir-format msgid "Invalid request" msgstr "" @@ -225,7 +225,7 @@ msgstr "" #: lib/pleroma/web/feed/tag_controller.ex:16 #: lib/pleroma/web/feed/user_controller.ex:69 #: lib/pleroma/web/o_status/o_status_controller.ex:132 -#: lib/pleroma/web/plugs/uploaded_media.ex:104 +#: lib/pleroma/web/plugs/uploaded_media.ex:84 #, elixir-autogen, elixir-format msgid "Not found" msgstr "" @@ -235,7 +235,7 @@ msgstr "" msgid "Poll's author can't vote" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:499 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:511 #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51 @@ -341,7 +341,7 @@ msgstr "" msgid "CAPTCHA expired" msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:77 +#: lib/pleroma/web/plugs/uploaded_media.ex:57 #, elixir-autogen, elixir-format msgid "Failed" msgstr "" @@ -361,7 +361,7 @@ msgstr "" msgid "Insufficient permissions: %{permissions}." msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:131 +#: lib/pleroma/web/plugs/uploaded_media.ex:111 #, elixir-autogen, elixir-format msgid "Internal Error" msgstr "" @@ -557,7 +557,7 @@ msgstr "" msgid "Access denied" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:331 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:343 #, elixir-autogen, elixir-format msgid "This API requires an authenticated user" msgstr "" @@ -567,7 +567,7 @@ msgstr "" msgid "User is not an admin." msgstr "" -#: lib/pleroma/user/backup.ex:73 +#: lib/pleroma/user/backup.ex:78 #, elixir-format msgid "Last export was less than a day ago" msgid_plural "Last export was less than %{days} days ago" @@ -607,3 +607,23 @@ msgstr "" #, elixir-autogen, elixir-format msgid "User isn't privileged." msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#, elixir-autogen, elixir-format +msgid "Bio is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:270 +#, elixir-autogen, elixir-format +msgid "Name is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:273 +#, elixir-autogen, elixir-format +msgid "One or more field entries are too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:276 +#, elixir-autogen, elixir-format +msgid "Too many field entries" +msgstr ""diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot@@ -219,3 +219,43 @@ msgstr "" #, elixir-autogen, elixir-format msgid "read:mutes" msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "push" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:backups" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:media" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:reports" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:follow" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:reports" +msgstr ""diff --git a/priv/repo/migrations/20220203224011_create_rules.exs b/priv/repo/migrations/20220203224011_create_rules.exs@@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.CreateRules do + use Ecto.Migration + + def change do + create_if_not_exists table(:rules) do + add(:priority, :integer, default: 0, null: false) + add(:text, :text, null: false) + + timestamps() + end + end +enddiff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs@@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'status' + """ + |> execute() + end + + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'status' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll', + 'update' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +enddiff --git a/priv/repo/migrations/20240406000000_add_hint_to_rules.exs b/priv/repo/migrations/20240406000000_add_hint_to_rules.exs@@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddHintToRules do + use Ecto.Migration + + def change do + alter table(:rules) do + add_if_not_exists(:hint, :text) + end + end +enddiff --git a/priv/repo/migrations/20240527144418_oban_queues_refactor.exs b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs@@ -0,0 +1,32 @@ +defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do + use Ecto.Migration + + @changed_queues [ + {"attachments_cleanup", "slow"}, + {"mailer", "background"}, + {"mute_expire", "background"}, + {"poll_notifications", "background"}, + {"activity_expiration", "slow"}, + {"filter_expiration", "background"}, + {"token_expiration", "background"}, + {"remote_fetcher", "background"}, + {"rich_media_expiration", "background"} + ] + + def up do + Enum.each(@changed_queues, fn {old, new} -> + execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';") + end) + + # Handled special as reverting this would not be ideal and leaving it is harmless + execute( + "UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';" + ) + end + + def down do + # Just move all slow queue jobs to background queue if we are reverting + # as the slow queue will not be processing jobs + execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';") + end +enddiff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl@@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then " else - SCRIPT=$(readlink -f "$0") + SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") FULL_ARGS="$*"diff --git a/supplemental/search/fastembed-api/Dockerfile b/supplemental/search/fastembed-api/Dockerfile@@ -0,0 +1,9 @@ +FROM python:3.9 + +WORKDIR /code +COPY fastembed-server.py /workdir/fastembed-server.py +COPY requirements.txt /workdir/requirements.txt + +RUN pip install -r /workdir/requirements.txt + +CMD ["python", "/workdir/fastembed-server.py"]diff --git a/supplemental/search/fastembed-api/README.md b/supplemental/search/fastembed-api/README.md@@ -0,0 +1,6 @@ +# About +This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend. + +# Usage + +The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds.diff --git a/supplemental/search/fastembed-api/compose.yml b/supplemental/search/fastembed-api/compose.yml@@ -0,0 +1,5 @@ +services: + web: + build: . + ports: + - "11345:11345"diff --git a/supplemental/search/fastembed-api/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py@@ -0,0 +1,27 @@ +from fastembed import TextEmbedding +from fastapi import FastAPI +from pydantic import BaseModel + +models = {} + +app = FastAPI() + +class EmbeddingRequest(BaseModel): + model: str + input: str + +@app.post("/v1/embeddings") +def embeddings(request: EmbeddingRequest): + model = models.get(request.model) or TextEmbedding(request.model) + models[request.model] = model + embeddings = next(model.embed(request.input)).tolist() + return {"data": [{"embedding": embeddings}]} + +@app.get("/health") +def health(): + return {"status": "ok"} + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=11345)diff --git a/supplemental/search/fastembed-api/requirements.txt b/supplemental/search/fastembed-api/requirements.txt@@ -0,0 +1,4 @@ +fastapi==0.111.0 +fastembed==0.2.7 +pydantic==1.10.15 +uvicorn==0.29.0diff --git a/test/fixtures/tesla_mock/gleasonator.com_host_meta b/test/fixtures/tesla_mock/gleasonator.com_host_meta@@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0"> + <Link rel="lrdd" template="https://gleasonator.com/.well-known/webfinger?resource={uri}" type="application/xrd+xml" /> +</XRD> +\ No newline at end of filediff --git a/test/fixtures/tesla_mock/webfinger_spoof.json b/test/fixtures/tesla_mock/webfinger_spoof.json@@ -0,0 +1,28 @@ +{ + "aliases": [ + "https://gleasonator.com/users/alex", + "https://mitra.social/users/alex" + ], + "links": [ + { + "href": "https://gleasonator.com/users/alex", + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/activity+json" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}" + } + ], + "subject": "acct:trump@whitehouse.gov" +}diff --git a/test/fixtures/webfinger/graf-imposter-webfinger.json b/test/fixtures/webfinger/graf-imposter-webfinger.json@@ -0,0 +1,41 @@ +{ + "subject": "acct:graf@poa.st", + "aliases": [ + "https://fba.ryona.agenc/webfingertest" + ], + "links": [ + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "self", + "type": "application/activity+json", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://fba.ryona.agenc/contact/follow?url={uri}" + }, + { + "rel": "http://schemas.google.com/g/2010#updates-from", + "type": "application/atom+xml", + "href": "" + }, + { + "rel": "salmon", + "href": "https://fba.ryona.agenc/salmon/friendica" + }, + { + "rel": "http://microformats.org/profile/hcard", + "type": "text/html", + "href": "https://fba.ryona.agenc/hcard/friendica" + }, + { + "rel": "http://joindiaspora.com/seed_location", + "type": "text/html", + "href": "https://fba.ryona.agenc" + } + ] +}diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs@@ -6,7 +6,6 @@ defmodule Pleroma.NotificationTest do use Pleroma.DataCase, async: false import Pleroma.Factory - import Mock alias Pleroma.FollowingRelationship alias Pleroma.Notification @@ -18,8 +17,6 @@ defmodule Pleroma.NotificationTest do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.NotificationView - alias Pleroma.Web.Push - alias Pleroma.Web.Streamer setup do Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config) @@ -115,6 +112,7 @@ defmodule Pleroma.NotificationTest do {:ok, [notification]} = Notification.create_notifications(status) assert notification.user_id == subscriber.id + assert notification.type == "status" end test "does not create a notification for subscribed users if status is a reply" do @@ -139,6 +137,21 @@ defmodule Pleroma.NotificationTest do assert Enum.empty?(subscriber_notifications) end + test "does not create subscriber notification if mentioned" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + + {:ok, status} = CommonAPI.post(user, %{status: "mentioning @#{subscriber.nickname}"}) + {:ok, [notification] = notifications} = Notification.create_notifications(status) + + assert length(notifications) == 1 + + assert notification.user_id == subscriber.id + assert notification.type == "mention" + end + test "it sends edited notifications to those who repeated a status" do user = insert(:user) repeated_user = insert(:user) @@ -175,158 +188,7 @@ defmodule Pleroma.NotificationTest do assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id) end - describe "CommonApi.post/2 notification-related functionality" do - test_with_mock "creates but does NOT send notification to blocker user", - Push, - [:passthrough], - [] do - user = insert(:user) - blocker = insert(:user) - {:ok, _user_relationship} = User.block(blocker, user) - - {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{blocker.nickname}!"}) - - blocker_id = blocker.id - assert [%Notification{user_id: ^blocker_id}] = Repo.all(Notification) - refute called(Push.send(:_)) - end - - test_with_mock "creates but does NOT send notification to notification-muter user", - Push, - [:passthrough], - [] do - user = insert(:user) - muter = insert(:user) - {:ok, _user_relationships} = User.mute(muter, user) - - {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{muter.nickname}!"}) - - muter_id = muter.id - assert [%Notification{user_id: ^muter_id}] = Repo.all(Notification) - refute called(Push.send(:_)) - end - - test_with_mock "creates but does NOT send notification to thread-muter user", - Push, - [:passthrough], - [] do - user = insert(:user) - thread_muter = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{thread_muter.nickname}!"}) - - {:ok, _} = CommonAPI.add_mute(thread_muter, activity) - - {:ok, _same_context_activity} = - CommonAPI.post(user, %{ - status: "hey-hey-hey @#{thread_muter.nickname}!", - in_reply_to_status_id: activity.id - }) - - [pre_mute_notification, post_mute_notification] = - Repo.all(from(n in Notification, where: n.user_id == ^thread_muter.id, order_by: n.id)) - - pre_mute_notification_id = pre_mute_notification.id - post_mute_notification_id = post_mute_notification.id - - assert called( - Push.send( - :meck.is(fn - %Notification{id: ^pre_mute_notification_id} -> true - _ -> false - end) - ) - ) - - refute called( - Push.send( - :meck.is(fn - %Notification{id: ^post_mute_notification_id} -> true - _ -> false - end) - ) - ) - end - end - describe "create_notification" do - @tag needs_streamer: true - test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do - %{user: user, token: oauth_token} = oauth_access(["read"]) - - task = - Task.async(fn -> - {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token) - assert_receive {:render_with_user, _, _, _, _}, 4_000 - end) - - task_user_notification = - Task.async(fn -> - {:ok, _topic} = - Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) - - assert_receive {:render_with_user, _, _, _, _}, 4_000 - end) - - activity = insert(:note_activity) - - notify = Notification.create_notification(activity, user) - assert notify.user_id == user.id - Task.await(task) - Task.await(task_user_notification) - end - - test "it creates a notification for user if the user blocks the activity author" do - activity = insert(:note_activity) - author = User.get_cached_by_ap_id(activity.data["actor"]) - user = insert(:user) - {:ok, _user_relationship} = User.block(user, author) - - assert Notification.create_notification(activity, user) - end - - test "it creates a notification for the user if the user mutes the activity author" do - muter = insert(:user) - muted = insert(:user) - {:ok, _} = User.mute(muter, muted) - muter = Repo.get(User, muter.id) - {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"}) - - notification = Notification.create_notification(activity, muter) - - assert notification.id - assert notification.seen - end - - test "notification created if user is muted without notifications" do - muter = insert(:user) - muted = insert(:user) - - {:ok, _user_relationships} = User.mute(muter, muted, %{notifications: false}) - - {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"}) - - assert Notification.create_notification(activity, muter) - end - - test "it creates a notification for an activity from a muted thread" do - muter = insert(:user) - other_user = insert(:user) - {:ok, activity} = CommonAPI.post(muter, %{status: "hey"}) - CommonAPI.add_mute(muter, activity) - - {:ok, activity} = - CommonAPI.post(other_user, %{ - status: "Hi @#{muter.nickname}", - in_reply_to_status_id: activity.id - }) - - notification = Notification.create_notification(activity, muter) - - assert notification.id - assert notification.seen - end - test "it disables notifications from strangers" do follower = insert(:user) @@ -603,9 +465,7 @@ defmodule Pleroma.NotificationTest do status: "hey yet again @#{other_user.nickname}!" }) - [_, read_notification] = Notification.set_read_up_to(other_user, n2.id) - - assert read_notification.activity.object + Notification.set_read_up_to(other_user, n2.id) [n3, n2, n1] = Notification.for_user(other_user) @@ -680,7 +540,7 @@ defmodule Pleroma.NotificationTest do status: "hey @#{other_user.nickname}!" }) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user in enabled_receivers end @@ -712,7 +572,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = Transmogrifier.handle_incoming(create_activity) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user in enabled_receivers end @@ -739,7 +599,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = Transmogrifier.handle_incoming(create_activity) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user not in enabled_receivers end @@ -756,8 +616,7 @@ defmodule Pleroma.NotificationTest do {:ok, activity_two} = CommonAPI.favorite(third_user, activity_one.id) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(activity_two) + enabled_receivers = Notification.get_notified_from_activity(activity_two) assert other_user not in enabled_receivers end @@ -779,7 +638,7 @@ defmodule Pleroma.NotificationTest do |> Map.put("to", [other_user.ap_id | like_data["to"]]) |> ActivityPub.persist(local: true) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(like) + enabled_receivers = Notification.get_notified_from_activity(like) assert other_user not in enabled_receivers end @@ -796,39 +655,36 @@ defmodule Pleroma.NotificationTest do {:ok, activity_two} = CommonAPI.repeat(activity_one.id, third_user) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(activity_two) + enabled_receivers = Notification.get_notified_from_activity(activity_two) assert other_user not in enabled_receivers end - test "it returns blocking recipient in disabled recipients list" do + test "it does not return blocking recipient in recipients list" do user = insert(:user) other_user = insert(:user) {:ok, _user_relationship} = User.block(other_user, user) {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end - test "it returns notification-muting recipient in disabled recipients list" do + test "it does not return notification-muting recipient in recipients list" do user = insert(:user) other_user = insert(:user) {:ok, _user_relationships} = User.mute(other_user, user) {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end - test "it returns thread-muting recipient in disabled recipients list" do + test "it does not return thread-muting recipient in recipients list" do user = insert(:user) other_user = insert(:user) @@ -842,14 +698,12 @@ defmodule Pleroma.NotificationTest do in_reply_to_status_id: activity.id }) - {enabled_receivers, disabled_receivers} = - Notification.get_notified_from_activity(same_context_activity) + enabled_receivers = Notification.get_notified_from_activity(same_context_activity) - assert [other_user] == disabled_receivers refute other_user in enabled_receivers end - test "it returns non-following domain-blocking recipient in disabled recipients list" do + test "it does not return non-following domain-blocking recipient in recipients list" do blocked_domain = "blocked.domain" user = insert(:user, %{ap_id: "https://#{blocked_domain}/@actor"}) other_user = insert(:user) @@ -858,10 +712,9 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end test "it returns following domain-blocking recipient in enabled recipients list" do @@ -874,10 +727,9 @@ defmodule Pleroma.NotificationTest do {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [other_user] == enabled_receivers - assert [] == disabled_receivers end test "it sends edited notifications to those who repeated a status" do @@ -897,11 +749,10 @@ defmodule Pleroma.NotificationTest do status: "hey @#{other_user.nickname}! mew mew" }) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(edit_activity) + enabled_receivers = Notification.get_notified_from_activity(edit_activity) assert repeated_user in enabled_receivers - assert other_user not in enabled_receivers + refute other_user in enabled_receivers end end @@ -1008,22 +859,6 @@ defmodule Pleroma.NotificationTest do assert Enum.empty?(Notification.for_user(user)) end - test "replying to a deleted post without tagging does not generate a notification" do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "test post"}) - {:ok, _deletion_activity} = CommonAPI.delete(activity.id, user) - - {:ok, _reply_activity} = - CommonAPI.post(other_user, %{ - status: "test reply", - in_reply_to_status_id: activity.id - }) - - assert Enum.empty?(Notification.for_user(user)) - end - test "notifications are deleted if a local user is deleted" do user = insert(:user) other_user = insert(:user) @@ -1189,13 +1024,13 @@ defmodule Pleroma.NotificationTest do assert Notification.for_user(user) == [] end - test "it returns notifications from a muted user when with_muted is set", %{user: user} do + test "it doesn't return notifications from a muted user when with_muted is set", %{user: user} do muted = insert(:user) {:ok, _user_relationships} = User.mute(user, muted) {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"}) - assert length(Notification.for_user(user, %{with_muted: true})) == 1 + assert Enum.empty?(Notification.for_user(user, %{with_muted: true})) end test "it doesn't return notifications from a blocked user when with_muted is set", %{diff --git a/test/pleroma/rule_test.exs b/test/pleroma/rule_test.exs@@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RuleTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Repo + alias Pleroma.Rule + + test "getting a list of rules sorted by priority" do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule", priority: 2}) + %{id: id3} = Rule.create(%{text: "Third rule", priority: 1}) + + rules = + Rule.query() + |> Repo.all() + + assert [%{id: ^id1}, %{id: ^id3}, %{id: ^id2}] = rules + end + + test "creating rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + assert %{text: "Example rule"} = Rule.get(id) + end + + test "editing rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + Rule.update(%{text: "There are no rules", priority: 2}, id) + + assert %{text: "There are no rules", priority: 2} = Rule.get(id) + end + + test "deleting rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + Rule.delete(id) + + assert [] = + Rule.query() + |> Pleroma.Repo.all() + end + + test "getting rules by ids" do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule"}) + %{id: _id3} = Rule.create(%{text: "Third rule"}) + + rules = Rule.get([id1, id2]) + + assert Enum.all?(rules, &(&1.id in [id1, id2])) + assert length(rules) == 2 + end +enddiff --git a/test/pleroma/scheduled_activity_test.exs b/test/pleroma/scheduled_activity_test.exs@@ -31,8 +31,7 @@ defmodule Pleroma.ScheduledActivityTest do {:ok, sa1} = ScheduledActivity.create(user, attrs) {:ok, sa2} = ScheduledActivity.create(user, attrs) - jobs = - Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] enddiff --git a/test/pleroma/search/healthcheck_test.exs b/test/pleroma/search/healthcheck_test.exs@@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.HealthcheckTest do + use Pleroma.DataCase + + import Tesla.Mock + + alias Pleroma.Search.Healthcheck + + @good1 "http://good1.example.com/healthz" + @good2 "http://good2.example.com/health" + @bad "http://bad.example.com/healthy" + + setup do + mock(fn + %{method: :get, url: @good1} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @good2} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @bad} -> + %Tesla.Env{ + status: 503, + body: "" + } + end) + + :ok + end + + test "true for 200 responses" do + assert Healthcheck.check([@good1]) + assert Healthcheck.check([@good1, @good2]) + end + + test "false if any response is not a 200" do + refute Healthcheck.check([@bad]) + refute Healthcheck.check([@good1, @bad]) + end +enddiff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs@@ -0,0 +1,199 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.QdrantSearchTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + import Mox + + alias Pleroma.Search.QdrantSearch + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.SearchIndexingWorker + + describe "Qdrant search" do + test "returns the correct healthcheck endpoints" do + # No openai healthcheck URL + Config + |> expect(:get, 2, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url"}[key] + end) + + [health_endpoint] = QdrantSearch.healthcheck_endpoints() + + assert "https://qdrant.url/healthz" == health_endpoint + + # Set openai healthcheck URL + Config + |> expect(:get, 2, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[ + key + ] + end) + + [_, health_endpoint] = QdrantSearch.healthcheck_endpoints() + + assert "https://openai.url/health" == health_endpoint + end + + test "searches for a term by encoding it and sending it to qdrant" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + refute data["filter"] + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{}) + + assert results == [activity] + end + + test "for a given actor, ask for only relevant matches" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + + assert data["filter"] == %{ + "must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}] + } + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = + QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user}) + + assert results == [activity] + end + + test "indexes a public post on creation, deletes from the index on deletion" do + user = insert(:user) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://openai.url/v1/embeddings"} -> + send(self(), "posted_to_openai") + + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> + send(self(), "posted_to_qdrant") + + data = Jason.decode!(body) + %{"points" => [%{"vector" => vector, "payload" => payload}]} = data + + assert vector == [1, 2, 3] + assert payload["actor"] + assert payload["published_at"] + + Tesla.Mock.json("ok") + + %{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} -> + send(self(), "deleted_from_qdrant") + Tesla.Mock.json("ok") + end) + + Config + |> expect(:get, 6, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued( + worker: SearchIndexingWorker, + args: args + ) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received("posted_to_openai") + assert_received("posted_to_qdrant") + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + + assert_received("deleted_from_qdrant") + end + end +enddiff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs@@ -67,6 +67,14 @@ defmodule Pleroma.SignatureTest do end end + describe "get_actor_id/1" do + test "it returns actor id" do + ap_id = "https://mastodon.social/users/lambadalambda" + + assert Signature.get_actor_id(make_fake_conn(ap_id)) == {:ok, ap_id} + end + end + describe "sign/2" do test "it returns signature headers" do user =diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs@@ -0,0 +1,158 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFSTest do + use Pleroma.DataCase + + alias Pleroma.Uploaders.IPFS + alias Tesla.Multipart + + import ExUnit.CaptureLog + import Mock + import Mox + + alias Pleroma.UnstubbedConfigMock, as: Config + + describe "get_final_url" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + + test "it returns the final url for put_file" do + assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add" + end + + test "it returns the final url for delete_file" do + assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm" + end + end + + describe "get_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end) + |> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end) + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end) + + :ok + end + + test "it returns path to ipfs file with cid as subdomain" do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> + "https://{CID}.ipfs.mydomain.com" + end) + + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://testcid.ipfs.mydomain.com"} + } + end + + test "it returns path to ipfs file with cid as path" do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> + "https://ipfs.mydomain.com/ipfs/{CID}" + end) + + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://ipfs.mydomain.com/ipfs/testcid"} + } + end + end + + describe "put_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + file_upload = %Pleroma.Upload{ + name: "image-tet.jpg", + content_type: "image/jpeg", + path: "test_folder/image-tet.jpg", + tempfile: Path.absname("test/instance_static/add/shortcode.png") + } + + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(file_upload.tempfile) + + [file_upload: file_upload, mp: mp] + end + + test "save file", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> + {:ok, + %Tesla.Env{ + status: 200, + body: + "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" + }} + end do + assert IPFS.put_file(file_upload) == + {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}} + end + end + + test "returns error", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> + {:error, "IPFS Gateway upload failed"} + end do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"} + end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" + end + end + + test "returns error if JSON decode fails", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: "invalid"}} + end do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} + end) =~ + "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError" + end + end + + test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}} + end do + assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} + end + end + end + + describe "delete_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + + test_with_mock "deletes file", Pleroma.HTTP, + post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] -> + {:ok, %{status: 204}} + end do + assert :ok = IPFS.delete_file("image.jpg") + end + end +enddiff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs@@ -877,109 +877,19 @@ defmodule Pleroma.UserTest do setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true) test "for mastodon" do - Tesla.Mock.mock(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - - %{url: "https://sub.example.com/users/a/collections/featured"} -> - %Tesla.Env{ - status: 200, - body: - File.read!("test/fixtures/users_mock/masto_featured.json") - |> String.replace("{{domain}}", "sub.example.com") - |> String.replace("{{nickname}}", "a"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@mastodon.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.mastodon.example/users/a" + assert fetched_user.nickname == "a@mastodon.example" end test "for pleroma" do - Tesla.Mock.mock(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@pleroma.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.pleroma.example/users/a" + assert fetched_user.nickname == "a@pleroma.example" end end @@ -2894,6 +2804,20 @@ defmodule Pleroma.UserTest do end end + describe "get_familiar_followers/3" do + test "returns familiar followers for a pair of users" do + user1 = insert(:user) + %{id: id2} = user2 = insert(:user) + user3 = insert(:user) + _user4 = insert(:user) + + User.follow(user1, user2) + User.follow(user2, user3) + + assert [%{id: ^id2}] = User.get_familiar_followers(user3, user1) + end + end + describe "account endorsements" do test "it pins people" do user = insert(:user)diff --git a/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs@@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy + + test "it allows posts without mentions" do + user = insert(:user, local: false) + assert user.note_count == 0 + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it allows posts from users with followers, posts, and age" do + user = + insert(:user, + local: false, + follower_count: 1, + note_count: 1, + inserted_at: ~N[1970-01-01 00:00:00] + ) + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it allows posts from local users" do + user = insert(:user, local: true) + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it rejects posts with mentions from users without followers" do + user = insert(:user, local: false, follower_count: 0) + + message = %{ + "type" => "Create", + "actor" => user.ap_id, + "object" => %{ + "to" => ["https://pleroma.soykaf.com/users/1"], + "cc" => ["https://pleroma.soykaf.com/users/1"], + "actor" => user.ap_id + } + } + + {:reject, _message} = AntiMentionSpamPolicy.filter(message) + end +enddiff --git a/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs@@ -0,0 +1,267 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicyTest do + use Pleroma.DataCase + + import ExUnit.CaptureLog + import Pleroma.Factory + + alias Pleroma.Constants + alias Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy + + require Pleroma.Constants + + @policy :mrf_nsfw_api + + @sfw_url "https://kittens.co/kitty.gif" + @nsfw_url "https://b00bies.com/nsfw.jpg" + @timeout_url "http://time.out/i.jpg" + + setup_all do + clear_config(@policy, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + ) + end + + setup do + Tesla.Mock.mock(fn + # NSFW URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@nsfw_url}"} -> + %Tesla.Env{status: 200, body: ~s({"score":0.99772077798843384,"url":"#{@nsfw_url}"})} + + # SFW URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@sfw_url}"} -> + %Tesla.Env{status: 200, body: ~s({"score":0.00011714912398019806,"url":"#{@sfw_url}"})} + + # Timeout URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@timeout_url}"} -> + {:error, :timeout} + + # Fallback URL + %{method: :get, url: "http://127.0.0.1:5000/?url=" <> url} -> + body = + ~s({"error_code":500,"error_reason":"[Errno -2] Name or service not known","url":"#{url}"}) + + %Tesla.Env{status: 500, body: body} + end) + + :ok + end + + describe "build_request_url/1" do + test "it works" do + expected = "http://127.0.0.1:5000/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + + test "it adds a trailing slash" do + clear_config([@policy, :url], "http://localhost:5000") + + expected = "http://localhost:5000/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + + test "it adds a trailing slash preserving the path" do + clear_config([@policy, :url], "http://localhost:5000/nsfw_api") + + expected = "http://localhost:5000/nsfw_api/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + end + + describe "parse_url/1" do + test "returns decoded JSON from the API server" do + expected = %{"score" => 0.99772077798843384, "url" => @nsfw_url} + assert NsfwApiPolicy.parse_url(@nsfw_url) == {:ok, expected} + end + + test "warns when the API server fails" do + expected = "[NsfwApiPolicy]: The API server failed. Skipping." + assert capture_log(fn -> NsfwApiPolicy.parse_url(@timeout_url) end) =~ expected + end + + test "returns {:error, _} tuple when the API server fails" do + capture_log(fn -> + assert {:error, _} = NsfwApiPolicy.parse_url(@timeout_url) + end) + end + end + + describe "check_url_nsfw/1" do + test "returns {:nsfw, _} tuple" do + expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(@nsfw_url) == expected + end + + test "returns {:sfw, _} tuple" do + expected = {:sfw, %{url: @sfw_url, score: 0.00011714912398019806, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(@sfw_url) == expected + end + + test "returns {:sfw, _} on failure" do + expected = {:sfw, %{url: @timeout_url, score: nil, threshold: 0.7}} + + capture_log(fn -> + assert NsfwApiPolicy.check_url_nsfw(@timeout_url) == expected + end) + end + + test "works with map URL" do + expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(%{"href" => @nsfw_url}) == expected + end + end + + describe "check_attachment_nsfw/1" do + test "returns {:nsfw, _} if any items are NSFW" do + attachment = %{"url" => [%{"href" => @nsfw_url}, @nsfw_url, @sfw_url]} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment} + end + + test "returns {:sfw, _} if all items are SFW" do + attachment = %{"url" => [%{"href" => @sfw_url}, @sfw_url, @sfw_url]} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:sfw, attachment} + end + + test "works with binary URL" do + attachment = %{"url" => @nsfw_url} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment} + end + end + + describe "check_object_nsfw/1" do + test "returns {:nsfw, _} if any items are NSFW" do + object = %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]} + assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object} + end + + test "returns {:sfw, _} if all items are SFW" do + object = %{"attachment" => [%{"url" => [%{"href" => @sfw_url}, @sfw_url]}]} + assert NsfwApiPolicy.check_object_nsfw(object) == {:sfw, object} + end + + test "works with embedded object" do + object = %{"object" => %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}} + assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object} + end + end + + describe "unlist/1" do + test "unlist addressing" do + user = insert(:user) + + object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address, "https://hello.world/users/alex"], + "actor" => user.ap_id + } + + expected = %{ + "to" => [user.follower_address], + "cc" => [Constants.as_public(), "https://hello.world/users/alex"], + "actor" => user.ap_id + } + + assert NsfwApiPolicy.unlist(object) == expected + end + + test "raise if user isn't found" do + object = %{ + "to" => [Constants.as_public()], + "cc" => [], + "actor" => "https://hello.world/users/alex" + } + + assert_raise(RuntimeError, fn -> + NsfwApiPolicy.unlist(object) + end) + end + end + + describe "mark_sensitive/1" do + test "adds nsfw tag and marks sensitive" do + object = %{"tag" => ["yolo"]} + expected = %{"tag" => ["yolo", "nsfw"], "sensitive" => true} + assert NsfwApiPolicy.mark_sensitive(object) == expected + end + + test "works with embedded object" do + object = %{"object" => %{"tag" => ["yolo"]}} + expected = %{"object" => %{"tag" => ["yolo", "nsfw"], "sensitive" => true}} + assert NsfwApiPolicy.mark_sensitive(object) == expected + end + end + + describe "filter/1" do + setup do + user = insert(:user) + + nsfw_object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address], + "actor" => user.ap_id, + "attachment" => [%{"url" => @nsfw_url}] + } + + sfw_object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address], + "actor" => user.ap_id, + "attachment" => [%{"url" => @sfw_url}] + } + + %{user: user, nsfw_object: nsfw_object, sfw_object: sfw_object} + end + + test "passes SFW object through", %{sfw_object: object} do + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW object through when actions are disabled", %{nsfw_object: object} do + clear_config([@policy, :mark_sensitive], false) + clear_config([@policy, :unlist], false) + clear_config([@policy, :reject], false) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW object through when :threshold is 1", %{nsfw_object: object} do + clear_config([@policy, :reject], true) + clear_config([@policy, :threshold], 1) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "rejects SFW object through when :threshold is 0", %{sfw_object: object} do + clear_config([@policy, :reject], true) + clear_config([@policy, :threshold], 0) + {:reject, _} = NsfwApiPolicy.filter(object) + end + + test "rejects NSFW when :reject is enabled", %{nsfw_object: object} do + clear_config([@policy, :reject], true) + {:reject, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW through when :reject is disabled", %{nsfw_object: object} do + clear_config([@policy, :reject], false) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "unlists NSFW when :unlist is enabled", %{user: user, nsfw_object: object} do + clear_config([@policy, :unlist], true) + {:ok, object} = NsfwApiPolicy.filter(object) + assert object["to"] == [user.follower_address] + end + + test "passes NSFW through when :unlist is disabled", %{nsfw_object: object} do + clear_config([@policy, :unlist], false) + {:ok, object} = NsfwApiPolicy.filter(object) + assert object["to"] == [Constants.as_public()] + end + end +enddiff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs@@ -27,19 +27,22 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do end test "works with honkerific attachments" do - attachment = %{ + honk = %{ "mediaType" => "", - "name" => "", - "summary" => "298p3RG7j27tfsZ9RQ.jpg", + "summary" => "Select your spirit chonk", + "name" => "298p3RG7j27tfsZ9RQ.jpg", "type" => "Document", "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg" } assert {:ok, attachment} = - AttachmentValidator.cast_and_validate(attachment) + honk + |> AttachmentValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) assert attachment.mediaType == "application/octet-stream" + assert attachment.summary == "Select your spirit chonk" + assert attachment.name == "298p3RG7j27tfsZ9RQ.jpg" end test "works with an unknown but valid mime type" dodiff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs@@ -827,31 +827,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, announce, _} = SideEffects.handle(announce) assert Repo.get_by(Notification, user_id: poster.id, activity_id: announce.id) end - - test "it streams out the announce", %{announce: announce} do - with_mocks([ - { - Pleroma.Web.Streamer, - [], - [ - stream: fn _, _ -> nil end - ] - }, - { - Pleroma.Web.Push, - [], - [ - send: fn _ -> nil end - ] - } - ]) do - {:ok, announce, _} = SideEffects.handle(announce) - - assert called(Pleroma.Web.Streamer.stream(["user", "list"], announce)) - - assert called(Pleroma.Web.Push.send(:_)) - end - end end describe "removing a follower" dodiff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs@@ -11,6 +11,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.ReportNote + alias Pleroma.Rule alias Pleroma.Web.CommonAPI setup do @@ -436,6 +437,34 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do "error" => "Invalid credentials." } end + + test "returns reports with specified role_id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + + %{id: rule_id} = Rule.create(%{text: "Example rule"}) + + rule_id = to_string(rule_id) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "", + rule_ids: [rule_id] + }) + + {:ok, _report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "" + }) + + response = + conn + |> get("/api/pleroma/admin/reports?rule_id=#{rule_id}") + |> json_response_and_validate_schema(:ok) + + assert %{"reports" => [%{"id" => ^report_id}]} = response + end end describe "POST /api/pleroma/admin/reports/:id/notes" dodiff --git a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs@@ -0,0 +1,82 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleControllerTest do + use Pleroma.Web.ConnCase, async: true + + import Pleroma.Factory + + alias Pleroma.Rule + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/pleroma/admin/rules" do + test "sorts rules by priority", %{conn: conn} do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule", priority: 2}) + %{id: id3} = Rule.create(%{text: "Third rule", priority: 1}) + + id1 = to_string(id1) + id2 = to_string(id2) + id3 = to_string(id3) + + response = + conn + |> get("/api/pleroma/admin/rules") + |> json_response_and_validate_schema(:ok) + + assert [%{"id" => ^id1}, %{"id" => ^id3}, %{"id" => ^id2}] = response + end + end + + describe "POST /api/pleroma/admin/rules" do + test "creates a rule", %{conn: conn} do + %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/rules", %{text: "Example rule"}) + |> json_response_and_validate_schema(:ok) + + assert %{text: "Example rule"} = Rule.get(id) + end + end + + describe "PATCH /api/pleroma/admin/rules" do + test "edits a rule", %{conn: conn} do + %{id: id} = Rule.create(%{text: "Example rule"}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/rules/#{id}", %{text: "There are no rules", priority: 2}) + |> json_response_and_validate_schema(:ok) + + assert %{text: "There are no rules", priority: 2} = Rule.get(id) + end + end + + describe "DELETE /api/pleroma/admin/rules" do + test "deletes a rule", %{conn: conn} do + %{id: id} = Rule.create(%{text: "Example rule"}) + + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/rules/#{id}") + |> json_response_and_validate_schema(:ok) + + assert [] = + Rule.query() + |> Pleroma.Repo.all() + end + end +enddiff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs@@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do import Pleroma.Factory + alias Pleroma.Rule alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.AdminAPI.ReportView @@ -38,7 +39,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do statuses: [], notes: [], state: "open", - id: activity.id + id: activity.id, + rules: [] } result = @@ -76,7 +78,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do statuses: [StatusView.render("show.json", %{activity: activity})], state: "open", notes: [], - id: report_activity.id + id: report_activity.id, + rules: [] } result = @@ -168,4 +171,22 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do assert report2.id == rendered |> Enum.at(0) |> Map.get(:id) assert report1.id == rendered |> Enum.at(1) |> Map.get(:id) end + + test "renders included rules" do + user = insert(:user) + other_user = insert(:user) + + %{id: rule_id, text: text} = Rule.create(%{text: "Example rule"}) + + rule_id = to_string(rule_id) + + {:ok, activity} = + CommonAPI.report(user, %{ + account_id: other_user.id, + rule_ids: [rule_id] + }) + + assert %{rules: [%{id: ^rule_id, text: ^text}]} = + ReportView.render("show.json", Report.extract_report_info(activity)) + end enddiff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs@@ -12,6 +12,7 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Rule alias Pleroma.UnstubbedConfigMock, as: ConfigMock alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -1363,6 +1364,33 @@ defmodule Pleroma.Web.CommonAPITest do assert first_report.data["state"] == "resolved" assert second_report.data["state"] == "resolved" end + + test "creates a report with provided rules" do + reporter = insert(:user) + target_user = insert(:user) + + %{id: rule_id} = Rule.create(%{text: "There are no rules"}) + + reporter_ap_id = reporter.ap_id + target_ap_id = target_user.ap_id + + report_data = %{ + account_id: target_user.id, + rule_ids: [rule_id] + } + + assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data) + + assert %Activity{ + actor: ^reporter_ap_id, + data: %{ + "type" => "Flag", + "object" => [^target_ap_id], + "state" => "open", + "rules" => [^rule_id] + } + } = flag_activity + end end describe "reblog muting" dodiff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs@@ -2172,6 +2172,55 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do end end + describe "familiar followers" do + setup do: oauth_access(["read:follows"]) + + test "fetch user familiar followers", %{user: user, conn: conn} do + %{id: id1} = other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [%{"id" => ^id1}], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + + test "returns empty array if followers are hidden", %{user: user, conn: conn} do + other_user1 = insert(:user, hide_follows: true) + %{id: id2} = other_user2 = insert(:user) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + + test "it respects hide_followers", %{user: user, conn: conn} do + other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user, hide_followers: true) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + end + describe "remove from followers" do setup do: oauth_access(["follow"])diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do # TODO: Should not need Cachex use Pleroma.Web.ConnCase + alias Pleroma.Rule alias Pleroma.User import Pleroma.Factory @@ -40,7 +41,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do "banner_upload_limit" => _, "background_image" => from_config_background, "shout_limit" => _, - "description_limit" => _ + "description_limit" => _, + "rules" => _ } = result assert result["pleroma"]["metadata"]["account_activation_required"] != nil @@ -125,4 +127,29 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do assert get(conn, "/api/v2/instance") |> json_response_and_validate_schema(200) end + + test "get instance rules", %{conn: conn} do + Rule.create(%{text: "Example rule", hint: "Rule description", priority: 1}) + Rule.create(%{text: "Third rule", priority: 2}) + Rule.create(%{text: "Second rule", priority: 1}) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + + assert [ + %{ + "text" => "Example rule", + "hint" => "Rule description" + }, + %{ + "text" => "Second rule", + "hint" => "" + }, + %{ + "text" => "Third rule", + "hint" => "" + } + ] = result["rules"] + end enddiff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do alias Pleroma.Activity alias Pleroma.Repo + alias Pleroma.Rule alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -81,6 +82,44 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do |> json_response_and_validate_schema(200) end + test "submit a report with rule_ids", %{ + conn: conn, + target_user: target_user + } do + %{id: rule_id} = Rule.create(%{text: "There are no rules"}) + + rule_id = to_string(rule_id) + + assert %{"action_taken" => false, "id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/reports", %{ + "account_id" => target_user.id, + "forward" => "false", + "rule_ids" => [rule_id] + }) + |> json_response_and_validate_schema(200) + + assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id) + end + + test "rules field is empty if provided wrong rule id", %{ + conn: conn, + target_user: target_user + } do + assert %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/reports", %{ + "account_id" => target_user.id, + "forward" => "false", + "rule_ids" => ["-1"] + }) + |> json_response_and_validate_schema(200) + + assert %Activity{data: %{"rules" => []}} = Activity.get_report(id) + end + test "account_id is required", %{ conn: conn, activity: activitydiff --git a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs@@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.Web.ConnCase, async: true alias Pleroma.Repo @@ -78,7 +79,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing")) assert job.args == %{"activity_id" => scheduled_activity.id} assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at) @@ -124,9 +125,11 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) - - assert job.args == %{"activity_id" => scheduled_activity.id} + assert_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id}, + queue: :federator_outgoing + ) res_conn = conn @@ -135,7 +138,11 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do assert %{} = json_response_and_validate_schema(res_conn, 200) refute Repo.get(ScheduledActivity, scheduled_activity.id) - refute Repo.get(Oban.Job, job.id) + + refute_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id} + ) res_conn = conndiff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs@@ -235,6 +235,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert Activity.get_in_reply_to_activity(activity).id == replied_to.id end + test "replying to a deleted status", %{user: user, conn: conn} do + {:ok, status} = CommonAPI.post(user, %{status: "cofe"}) + {:ok, _deleted_status} = CommonAPI.delete(status.id, user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses", %{"status" => "xD", "in_reply_to_id" => status.id}) + |> json_response_and_validate_schema(422) + end + test "replying to a direct message with visibility other than direct", %{ user: user, conn: conndiff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs@@ -331,4 +331,31 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do test_notifications_rendering([notification], user, [expected]) end + + test "Subscribed status notification" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hi"}) + {:ok, [notification]} = Notification.create_notifications(activity) + + user = User.get_cached_by_id(user.id) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "status", + account: + AccountView.render("show.json", %{ + user: user, + for: subscriber + }), + status: StatusView.render("show.json", %{activity: activity, for: subscriber}), + created_at: Utils.to_masto_date(notification.inserted_at) + } + + test_notifications_rendering([notification], subscriber, [expected]) + end enddiff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs@@ -591,45 +591,78 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert mention.url == recipient.ap_id end - test "attachments" do - object = %{ - "type" => "Image", - "url" => [ - %{ - "mediaType" => "image/png", - "href" => "someurl", - "width" => 200, - "height" => 100 - } - ], - "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", - "uuid" => 6 - } + describe "attachments" do + test "Complete Mastodon style" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl", + "width" => 200, + "height" => 100 + } + ], + "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", + "uuid" => 6 + } - expected = %{ - id: "1638338801", - type: "image", - url: "someurl", - remote_url: "someurl", - preview_url: "someurl", - text_url: "someurl", - description: nil, - pleroma: %{mime_type: "image/png"}, - meta: %{original: %{width: 200, height: 100, aspect: 2}}, - blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" - } + expected = %{ + id: "1638338801", + type: "image", + url: "someurl", + remote_url: "someurl", + preview_url: "someurl", + text_url: "someurl", + description: nil, + pleroma: %{mime_type: "image/png"}, + meta: %{original: %{width: 200, height: 100, aspect: 2}}, + blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" + } - api_spec = Pleroma.Web.ApiSpec.spec() + api_spec = Pleroma.Web.ApiSpec.spec() - assert expected == StatusView.render("attachment.json", %{attachment: object}) - assert_schema(expected, "Attachment", api_spec) + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) - # If theres a "id", use that instead of the generated one - object = Map.put(object, "id", 2) - result = StatusView.render("attachment.json", %{attachment: object}) + # If theres a "id", use that instead of the generated one + object = Map.put(object, "id", 2) + result = StatusView.render("attachment.json", %{attachment: object}) - assert %{id: "2"} = result - assert_schema(result, "Attachment", api_spec) + assert %{id: "2"} = result + assert_schema(result, "Attachment", api_spec) + end + + test "Honkerific" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl" + } + ], + "name" => "fool.jpeg", + "summary" => "they have played us for absolute fools." + } + + expected = %{ + blurhash: nil, + description: "they have played us for absolute fools.", + id: "1638338801", + pleroma: %{mime_type: "image/png", name: "fool.jpeg"}, + preview_url: "someurl", + remote_url: "someurl", + text_url: "someurl", + type: "image", + url: "someurl" + } + + api_spec = Pleroma.Web.ApiSpec.spec() + + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) + end end test "put the url advertised in the Activity in to the url attribute" dodiff --git a/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs@@ -21,13 +21,11 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do {:ok, [notification1]} = Notification.create_notifications(activity1) {:ok, [notification2]} = Notification.create_notifications(activity2) - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id}) - |> json_response_and_validate_schema(:ok) + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id}) + |> json_response_and_validate_schema(:ok) - assert %{"pleroma" => %{"is_seen" => true}} = response assert Repo.get(Notification, notification1.id).seen refute Repo.get(Notification, notification2.id).seen end @@ -40,14 +38,17 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3}) - [response1, response2] = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id}) - |> json_response_and_validate_schema(:ok) + refute Repo.get(Notification, notification1.id).seen + refute Repo.get(Notification, notification2.id).seen + refute Repo.get(Notification, notification3.id).seen + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id}) + |> json_response_and_validate_schema(:ok) + + [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3}) - assert %{"pleroma" => %{"is_seen" => true}} = response1 - assert %{"pleroma" => %{"is_seen" => true}} = response2 assert Repo.get(Notification, notification1.id).seen assert Repo.get(Notification, notification2.id).seen refute Repo.get(Notification, notification3.id).seendiff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs@@ -3,14 +3,52 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true alias Plug.Conn + import Mox + + setup do + base_config = Pleroma.Config.get([:http_security]) + %{base_config: base_config} + end + + defp mock_config(config, additional \\ %{}) do + Pleroma.StaticStubbedConfigMock + |> stub(:get, fn + [:http_security, key] -> config[key] + key -> additional[key] + end) + end + describe "http security enabled" do - setup do: clear_config([:http_security, :enabled], true) + setup %{base_config: base_config} do + %{base_config: Keyword.put(base_config, :enabled, true)} + end + + test "it does not contain unsafe-eval", %{conn: conn, base_config: base_config} do + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + refute header =~ ~r/unsafe-eval/ + end + + test "with allow_unsafe_eval set, it does contain it", %{conn: conn, base_config: base_config} do + base_config = + base_config + |> Keyword.put(:allow_unsafe_eval, true) + + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + assert header =~ ~r/unsafe-eval/ + end - test "it sends CSP headers when enabled", %{conn: conn} do + test "it sends CSP headers when enabled", %{conn: conn, base_config: base_config} do + mock_config(base_config) conn = get(conn, "/api/v1/instance") refute Conn.get_resp_header(conn, "x-xss-protection") == [] @@ -22,8 +60,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do refute Conn.get_resp_header(conn, "content-security-policy") == [] end - test "it sends STS headers when enabled", %{conn: conn} do - clear_config([:http_security, :sts], true) + test "it sends STS headers when enabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, true) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -31,8 +71,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do refute Conn.get_resp_header(conn, "expect-ct") == [] end - test "it does not send STS headers when disabled", %{conn: conn} do - clear_config([:http_security, :sts], false) + test "it does not send STS headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, false) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -40,19 +82,30 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert Conn.get_resp_header(conn, "expect-ct") == [] end - test "referrer-policy header reflects configured value", %{conn: conn} do - resp = get(conn, "/api/v1/instance") + test "referrer-policy header reflects configured value", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"] - clear_config([:http_security, :referrer_policy], "no-referrer") + base_config + |> Keyword.put(:referrer_policy, "no-referrer") + |> mock_config resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"] end - test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do + test "it sends `report-to` & `report-uri` CSP response headers", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -65,7 +118,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" end - test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do + test "default values for img-src and media-src with disabled media proxy", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -73,60 +130,129 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert csp =~ "img-src 'self' data: blob: https:;" end - test "it sets the Service-Worker-Allowed header", %{conn: conn} do - clear_config([:http_security, :enabled], true) - clear_config([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) - - clear_config([:frontends, :available], %{ - "fedi-fe" => %{ - "name" => "fedi-fe", - "custom-http-headers" => [{"service-worker-allowed", "/"}] - } - }) - + test "it sets the Service-Worker-Allowed header", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, true) + + additional_config = + %{} + |> Map.put([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) + |> Map.put( + [:frontends, :available], + %{ + "fedi-fe" => %{ + "name" => "fedi-fe", + "custom-http-headers" => [{"service-worker-allowed", "/"}] + } + } + ) + + mock_config(base_config, additional_config) conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "service-worker-allowed") == ["/"] end end describe "img-src and media-src" do - setup do - clear_config([:http_security, :enabled], true) - clear_config([:media_proxy, :enabled], true) - clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false) + setup %{base_config: base_config} do + base_config = + base_config + |> Keyword.put(:enabled, true) + + additional_config = + %{} + |> Map.put([:media_proxy, :enabled], true) + |> Map.put([:media_proxy, :proxy_opts, :redirect_on_failure], false) + |> Map.put([:media_proxy, :whitelist], []) + + %{base_config: base_config, additional_config: additional_config} end - test "media_proxy with base_url", %{conn: conn} do + test "media_proxy with base_url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example.com" - clear_config([:media_proxy, :base_url], url) + + additional_config = + additional_config + |> Map.put([:media_proxy, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "upload with base url", %{conn: conn} do + test "upload with base url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example2.com" - clear_config([Pleroma.Upload, :base_url], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Upload, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "with S3 public endpoint", %{conn: conn} do + test "with S3 public endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example3.com" - clear_config([Pleroma.Uploaders.S3, :public_endpoint], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Uploaders.S3, :public_endpoint], url) + + mock_config(base_config, additional_config) assert_media_img_src(conn, url) end - test "with captcha endpoint", %{conn: conn} do - clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + test "with captcha endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + |> Map.put([Pleroma.Captcha, :method], Pleroma.Captcha.Mock) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://captcha.com") end - test "with media_proxy whitelist", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + test "with media_proxy whitelist", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://example7.com https://example6.com") end # TODO: delete after removing support bare domains for media proxy whitelist - test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + test "with media_proxy bare domains whitelist (deprecated)", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "example5.com example4.com") end end @@ -138,8 +264,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert csp =~ "img-src 'self' data: blob: #{url};" end - test "it does not send CSP headers when disabled", %{conn: conn} do - clear_config([:http_security, :enabled], false) + test "it does not send CSP headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, false) + |> mock_config conn = get(conn, "/api/v1/instance")diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs@@ -3,77 +3,89 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true + + alias Pleroma.StaticStubbedConfigMock, as: ConfigMock + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock alias Pleroma.Web.Plugs.HTTPSignaturePlug - import Plug.Conn + import Mox import Phoenix.Controller, only: [put_format: 2] - import Mock + import Plug.Conn - test "it call HTTPSignatures to check validity if the actor sighed it" do + test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) - with_mock HTTPSignatures, validate_conn: fn _ -> true end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> put_format("activity+json") - |> HTTPSignaturePlug.call(%{}) + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + conn = + conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end describe "requires a signature when `authorized_fetch_mode` is enabled" do setup do - clear_config([:activitypub, :authorized_fetch_mode], true) - params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json") [conn: conn] end - test "when signature header is present", %{conn: conn} do - with_mock HTTPSignatures, validate_conn: fn _ -> false end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == false - assert conn.halted == true - assert conn.status == 401 - assert conn.state == :sent - assert conn.resp_body == "Request not signed" - assert called(HTTPSignatures.validate_conn(:_)) - end - - with_mock HTTPSignatures, validate_conn: fn _ -> true end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + test "when signature header is present", %{conn: orig_conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) + + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) + + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == false + assert conn.halted == true + assert conn.status == 401 + assert conn.state == :sent + assert conn.resp_body == "Request not signed" + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end test "halts the connection when `signature` header is not present", %{conn: conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) + conn = HTTPSignaturePlug.call(conn, %{}) assert conn.assigns[:valid_signature] == nil assert conn.halted == true @@ -81,5 +93,73 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do assert conn.state == :sent assert conn.resp_body == "Request not signed" end + + test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> + ["192.168.0.0/24"] + end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) + + conn = + conn + |> Map.put(:remote_ip, {192, 168, 0, 1}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.remote_ip == {192, 168, 0, 1} + assert conn.halted == false + end + end + + test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == true + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://allowed.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end enddiff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs@@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do alias Pleroma.UnstubbedConfigMock, as: ConfigMock alias Pleroma.Web.RichMedia.Card + alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl setup do ConfigMock @@ -82,6 +83,12 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till end + test "AWS URL for an image without expiration works" do + og_data = %{"image" => "https://amazonaws.com/image.png"} + + assert is_nil(AwsSignedUrl.ttl(og_data, "")) + end + defp construct_s3_url(timestamp, valid_till) do "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" enddiff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs@@ -76,15 +76,6 @@ defmodule Pleroma.Web.WebFingerTest do {:ok, _data} = WebFinger.finger(user) end - test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do - user = "kaniini@gerzilla.de" - - {:ok, data} = WebFinger.finger(user) - - assert data["ap_id"] == "https://gerzilla.de/channel/kaniini" - assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}" - end - test "it work for AP-only user" do user = "kpherox@mstdn.jp" @@ -99,12 +90,6 @@ defmodule Pleroma.Web.WebFingerTest do assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}" end - test "it works for friendica" do - user = "lain@squeet.me" - - {:ok, _data} = WebFinger.finger(user) - end - test "it gets the xrd endpoint" do {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la") @@ -203,5 +188,44 @@ defmodule Pleroma.Web.WebFingerTest do assert :error = WebFinger.finger("pekorino@pawoo.net") end + + test "prevents spoofing" do + Tesla.Mock.mock(fn + %{ + url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com" + } -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"), + headers: [{"content-type", "application/jrd+json"}] + }} + + %{url: "https://gleasonator.com/.well-known/host-meta"} -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta") + }} + end) + + {:error, _data} = WebFinger.finger("alex@gleasonator.com") + end + end + + @tag capture_log: true + test "prevents forgeries" do + Tesla.Mock.mock(fn + %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} -> + fake_webfinger = + File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!() + + Tesla.Mock.json(fake_webfinger) + + %{url: "https://fba.ryona.agency/.well-known/host-meta"} -> + {:ok, %Tesla.Env{status: 404}} + end) + + assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency") end enddiff --git a/test/support/data_case.ex b/test/support/data_case.ex@@ -116,6 +116,7 @@ defmodule Pleroma.DataCase do Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator) Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config) Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig) + Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy) end def ensure_local_uploader(context) dodiff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex@@ -1521,6 +1521,120 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.mastodon.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.mastodon.example") + }} + end + + def get( + "https://sub.mastodon.example/.well-known/webfinger?resource=acct:a@mastodon.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "mastodon.example") + |> String.replace("{{subdomain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a/collections/featured", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "sub.mastodon.example") + |> String.replace("{{nickname}}", "a"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.pleroma.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.pleroma.example") + }} + end + + def get( + "https://sub.pleroma.example/.well-known/webfinger?resource=acct:a@pleroma.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "pleroma.example") + |> String.replace("{{subdomain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.pleroma.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}diff --git a/test/support/http_signatures_proxy.ex b/test/support/http_signatures_proxy.ex@@ -0,0 +1,9 @@ +defmodule Pleroma.Test.HTTPSignaturesProxy do + @behaviour Pleroma.HTTPSignaturesAPI + + @impl true + defdelegate validate_conn(conn), to: HTTPSignatures + + @impl true + defdelegate signature_for_conn(conn), to: HTTPSignatures +enddiff --git a/test/support/mocks.ex b/test/support/mocks.ex@@ -28,6 +28,7 @@ Mox.defmock(Pleroma.Web.FederatorMock, for: Pleroma.Web.Federator.Publishing) Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting) +Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI) Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging)