commit: 270f09dbc2cb145e5a06b72189bd1b91dd3d3a02 parent: b7fc722a2e9e93341229cb122aac605421782295 Author: kaniini <nenolod@gmail.com> Date: Fri, 14 Jun 2019 15:45:05 +0000 Merge branch 'feature/905-dynamic-configuration-in-sql' into 'develop' Feature/905 dynamic configuration in sql Closes #905 See merge request pleroma/pleroma!1195Diffstat:
32 files changed, 940 insertions(+), 52 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md @@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.database update_users_following_followers_counts` - Mix Tasks: `mix pleroma.user toggle_confirmed` +- Mix Tasks: `mix pleroma.config migrate_to_db` +- Mix Tasks: `mix pleroma.config migrate_from_db` - Federation: Support for `Question` and `Answer` objects - Federation: Support for reports - Configuration: `poll_limits` option @@ -37,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: added filters (role, tags, email, name) for users endpoint - Admin API: Endpoints for managing reports - Admin API: Endpoints for deleting and changing the scope of individual reported statuses +- Admin API: Endpoints to view and change config settings. - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/) - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension) diff --git a/config/config.exs b/config/config.exs @@ -245,7 +245,8 @@ config :pleroma, :instance, healthcheck: false, remote_post_retention_days: 90, skip_thread_containment: true, - limit_to_local_content: :unauthenticated + limit_to_local_content: :unauthenticated, + dynamic_configuration: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/config/dev.exs b/config/dev.exs @@ -59,3 +59,6 @@ else "!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs" ) end + +if File.exists?("./config/dev.migrated.secret.exs"), + do: import_config("./config/dev.migrated.secret.exs") diff --git a/config/prod.exs b/config/prod.exs @@ -63,3 +63,6 @@ config :logger, level: :warn # Finally import the config/prod.secret.exs # which should be versioned separately. import_config "prod.secret.exs" + +if File.exists?("./config/prod.migrated.secret.exs"), + do: import_config("./config/prod.migrated.secret.exs") diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md @@ -289,7 +289,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `limit`: optional, the number of records to retrieve - `since_id`: optional, returns results that are more recent than the specified id - `max_id`: optional, returns results that are older than the specified id -- Response: +- Response: - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin - On success: JSON, returns a list of reports, where: - `account`: the user who has been reported @@ -443,7 +443,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: - `id` - Response: - - On failure: + - On failure: - 403 Forbidden `{"error": "error_msg"}` - 404 Not Found `"Not found"` - On success: JSON, Report object (see above) @@ -454,8 +454,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: - `id` - `state`: required, the new state. Valid values are `open`, `closed` and `resolved` -- Response: - - On failure: +- Response: + - On failure: - 400 Bad Request `"Unsupported state"` - 403 Forbidden `{"error": "error_msg"}` - 404 Not Found `"Not found"` @@ -467,10 +467,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: - `id` - `status`: required, the message -- Response: - - On failure: - - 400 Bad Request `"Invalid parameters"` when `status` is missing - - 403 Forbidden `{"error": "error_msg"}` +- Response: + - On failure: + - 400 Bad Request `"Invalid parameters"` when `status` is missing + - 403 Forbidden `{"error": "error_msg"}` - 404 Not Found `"Not found"` - On success: JSON, created Mastodon Status entity @@ -540,10 +540,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `id` - `sensitive`: optional, valid values are `true` or `false` - `visibility`: optional, valid values are `public`, `private` and `unlisted` -- Response: - - On failure: +- Response: + - On failure: - 400 Bad Request `"Unsupported visibility"` - - 403 Forbidden `{"error": "error_msg"}` + - 403 Forbidden `{"error": "error_msg"}` - 404 Not Found `"Not found"` - On success: JSON, Mastodon Status entity @@ -552,8 +552,88 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Method `DELETE` - Params: - `id` -- Response: - - On failure: - - 403 Forbidden `{"error": "error_msg"}` +- Response: + - On failure: + - 403 Forbidden `{"error": "error_msg"}` - 404 Not Found `"Not found"` - On success: 200 OK `{}` + +## `/api/pleroma/admin/config` +### List config settings +- Method `GET` +- Params: none +- Response: + +```json +{ + configs: [ + { + "key": string, + "value": string or {} or [] + } + ] +} +``` + +## `/api/pleroma/admin/config` +### Update config settings +Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`. +Atom or boolean value can be passed with `:` in the beginning, e.g. `":true"`, `":upload"`. +Integer with `i:`, e.g. `"i:150"`. + +Compile time settings (need instance reboot): +- all settings by this keys: + - `:hackney_pools` + - `:chat` + - `Pleroma.Web.Endpoint` + - `Pleroma.Repo` +- part settings: + - `Pleroma.Captcha` -> `:seconds_valid` + - `Pleroma.Upload` -> `:proxy_remote` + - `:instance` -> `:upload_limit` + +- Method `POST` +- Params: + - `configs` => [ + - `key` (string) + - `value` (string, [], {}) + - `delete` = true (optional, if parameter must be deleted) + ] + +- Request (example): + +```json +{ + configs: [ + { + "key": "Pleroma.Upload", + "value": { + "uploader": "Pleroma.Uploaders.Local", + "filters": ["Pleroma.Upload.Filter.Dedupe"], + "link_name": ":true", + "proxy_remote": ":false", + "proxy_opts": { + "redirect_on_failure": ":false", + "max_body_length": "i:1048576", + "http": { + "follow_redirect": ":true", + "pool": ":upload" + } + } + } + } + ] +} + +- Response: + +```json +{ + configs: [ + { + "key": string, + "value": string or {} or [] + } + ] +} +``` diff --git a/docs/config.md b/docs/config.md @@ -114,6 +114,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. +* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. ## :logger diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex @@ -0,0 +1,68 @@ +defmodule Mix.Tasks.Pleroma.Config do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + alias Pleroma.Repo + alias Pleroma.Web.AdminAPI.Config + @shortdoc "Manages the location of the config" + @moduledoc """ + Manages the location of the config. + + ## Transfers config from file to DB. + + mix pleroma.config migrate_to_db + + ## Transfers config from DB to file. + + mix pleroma.config migrate_from_db ENV + """ + + def run(["migrate_to_db"]) do + Common.start_pleroma() + + if Pleroma.Config.get([:instance, :dynamic_configuration]) do + Application.get_all_env(:pleroma) + |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end) + |> Enum.each(fn {k, v} -> + key = to_string(k) |> String.replace("Elixir.", "") + {:ok, _} = Config.update_or_create(%{key: key, value: v}) + Mix.shell().info("#{key} is migrated.") + end) + + Mix.shell().info("Settings migrated.") + else + Mix.shell().info( + "Migration is not allowed by config. You can change this behavior in instance settings." + ) + end + end + + def run(["migrate_from_db", env]) do + Common.start_pleroma() + + if Pleroma.Config.get([:instance, :dynamic_configuration]) do + config_path = "config/#{env}.migrated.secret.exs" + + {:ok, file} = File.open(config_path, [:write]) + + Repo.all(Config) + |> Enum.each(fn config -> + mark = if String.starts_with?(config.key, "Pleroma."), do: ",", else: ":" + + IO.write( + file, + "config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n" + ) + + {:ok, _} = Repo.delete(config) + Mix.shell().info("#{config.key} deleted from DB.") + end) + + File.close(file) + System.cmd("mix", ["format", config_path]) + else + Mix.shell().info( + "Migration is not allowed by config. You can change this behavior in instance settings." + ) + end + end +end diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex @@ -55,15 +55,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do are extracted). """ - @default_manifest Pleroma.Config.get!([:emoji, :default_manifest]) - def run(["ls-packs" | args]) do Application.ensure_all_started(:hackney) {options, [], []} = parse_global_opts(args) manifest = - fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest) + fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest()) Enum.each(manifest, fn {name, info} -> to_print = [ @@ -88,7 +86,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do {options, pack_names, []} = parse_global_opts(args) - manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest + manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest() manifest = fetch_manifest(manifest_url) @@ -298,4 +296,6 @@ defmodule Mix.Tasks.Pleroma.Emoji do Tesla.client(middleware) end + + defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest]) end diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex @@ -30,6 +30,7 @@ defmodule Mix.Tasks.Pleroma.Instance do - `--dbuser DBUSER` - the user (aka role) to use for the database connection - `--dbpass DBPASS` - the password to use for the database connection - `--indexable Y/N` - Allow/disallow indexing site by search engines + - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part """ def run(["gen" | rest]) do @@ -48,7 +49,8 @@ defmodule Mix.Tasks.Pleroma.Instance do dbname: :string, dbuser: :string, dbpass: :string, - indexable: :string + indexable: :string, + db_configurable: :string ], aliases: [ o: :output, @@ -101,6 +103,14 @@ defmodule Mix.Tasks.Pleroma.Instance do "y" ) === "y" + db_configurable? = + Common.get_option( + options, + :db_configurable, + "Do you want to be able to configure instance from admin part? (y/n)", + "y" + ) === "y" + dbhost = Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost") @@ -144,7 +154,8 @@ defmodule Mix.Tasks.Pleroma.Instance do secret: secret, signing_salt: signing_salt, web_push_public_key: Base.url_encode64(web_push_public_key, padding: false), - web_push_private_key: Base.url_encode64(web_push_private_key, padding: false) + web_push_private_key: Base.url_encode64(web_push_private_key, padding: false), + db_configurable?: db_configurable? ) result_psql = diff --git a/lib/mix/tasks/pleroma/sample_config.eex b/lib/mix/tasks/pleroma/sample_config.eex @@ -16,7 +16,8 @@ config :pleroma, :instance, notify_email: "<%= notify_email %>", limit: 5000, registrations_open: true, - dedupe_media: false + dedupe_media: false, + dynamic_configuration: <%= db_configurable? %> config :pleroma, :media_proxy, enabled: false, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Application do [ # Start the Ecto repository %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor}, + %{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}}, %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}}, %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}}, %{ @@ -186,7 +187,7 @@ defmodule Pleroma.Application do else [] end ++ - if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do + if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do [:upload] else [] diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.Config.TransferTask do + use Task + alias Pleroma.Web.AdminAPI.Config + + def start_link do + load_and_update_env() + if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo) + :ignore + end + + def load_and_update_env do + if Pleroma.Config.get([:instance, :dynamic_configuration]) do + Pleroma.Repo.all(Config) + |> Enum.each(&update_env(&1)) + end + end + + defp update_env(setting) do + try do + key = + if String.starts_with?(setting.key, "Pleroma.") do + "Elixir." <> setting.key + else + setting.key + end + + Application.put_env( + :pleroma, + String.to_existing_atom(key), + Config.from_binary(setting.value) + ) + rescue + e -> + require Logger + + Logger.warn( + "updating env causes error, key: #{inspect(setting.key)}, error: #{inspect(e)}" + ) + end + end +end diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex @@ -22,7 +22,6 @@ defmodule Pleroma.Emoji do @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] - @groups Pleroma.Config.get([:emoji, :groups]) @doc false def start_link do @@ -87,6 +86,8 @@ defmodule Pleroma.Emoji do "emoji" ) + emoji_groups = Pleroma.Config.get([:emoji, :groups]) + case File.ls(emoji_dir_path) do {:error, :enoent} -> # The custom emoji directory doesn't exist, @@ -118,7 +119,7 @@ defmodule Pleroma.Emoji do emojis = Enum.flat_map( packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end + fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end ) true = :ets.insert(@ets, emojis) @@ -129,9 +130,9 @@ defmodule Pleroma.Emoji do shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) emojis = - (load_from_file("config/emoji.txt") ++ - load_from_file("config/custom_emoji.txt") ++ - load_from_globs(shortcode_globs)) + (load_from_file("config/emoji.txt", emoji_groups) ++ + load_from_file("config/custom_emoji.txt", emoji_groups) ++ + load_from_globs(shortcode_globs, emoji_groups)) |> Enum.reject(fn value -> value == nil end) true = :ets.insert(@ets, emojis) @@ -139,13 +140,13 @@ defmodule Pleroma.Emoji do :ok end - defp load_pack(pack_dir) do + defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) emoji_txt = Path.join(pack_dir, "emoji.txt") if File.exists?(emoji_txt) do - load_from_file(emoji_txt) + load_from_file(emoji_txt, emoji_groups) else Logger.info( "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" @@ -155,7 +156,7 @@ defmodule Pleroma.Emoji do |> Enum.map(fn {shortcode, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) - {shortcode, filename, [to_string(match_extra(@groups, filename))]} + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} end) end end @@ -184,21 +185,21 @@ defmodule Pleroma.Emoji do |> Enum.filter(fn f -> Path.extname(f) in exts end) end - defp load_from_file(file) do + defp load_from_file(file, emoji_groups) do if File.exists?(file) do - load_from_file_stream(File.stream!(file)) + load_from_file_stream(File.stream!(file), emoji_groups) else [] end end - defp load_from_file_stream(stream) do + defp load_from_file_stream(stream, emoji_groups) do stream |> Stream.map(&String.trim/1) |> Stream.map(fn line -> case String.split(line, ~r/,\s*/) do [name, file] -> - {name, file, [to_string(match_extra(@groups, file))]} + {name, file, [to_string(match_extra(emoji_groups, file))]} [name, file | tags] -> {name, file, tags} @@ -210,7 +211,7 @@ defmodule Pleroma.Emoji do |> Enum.to_list() end - defp load_from_globs(globs) do + defp load_from_globs(globs, emoji_groups) do static_path = Path.join(:code.priv_dir(:pleroma), "static") paths = @@ -221,7 +222,7 @@ defmodule Pleroma.Emoji do |> Enum.concat() Enum.map(paths, fn path -> - tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path))) + tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) shortcode = Path.basename(path, Path.extname(path)) external_path = Path.join("/", Path.relative_to(path, static_path)) {shortcode, external_path, [to_string(tag)]} diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Instances do def reachability_datetime_threshold do federation_reachability_timeout_days = - Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0 + Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0) if federation_reachability_timeout_days > 0 do NaiveDateTime.add( diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex @@ -36,7 +36,7 @@ defmodule Pleroma.Plugs.UploadedMedia do conn end - config = Pleroma.Config.get([Pleroma.Upload]) + config = Pleroma.Config.get(Pleroma.Upload) with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex @@ -146,7 +146,7 @@ defmodule Pleroma.ReverseProxy do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case :hackney.request(method, url, headers, "", hackney_opts) do + case hackney().request(method, url, headers, "", hackney_opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -196,7 +196,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- :hackney.stream_body(client), + {:ok, data} <- hackney().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), @@ -377,4 +377,6 @@ defmodule Pleroma.ReverseProxy do defp increase_read_duration(_) do {:ok, :no_duration_limit, :no_duration_limit} end + + defp hackney, do: Pleroma.Config.get(:hackney, :hackney) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex @@ -1036,9 +1036,7 @@ defmodule Pleroma.User do Pleroma.HTML.Scrubber.TwitterText end - @default_scrubbers Pleroma.Config.get([:markup, :scrub_policy]) - - def html_filter_policy(_), do: @default_scrubbers + def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy]) def fetch_by_ap_id(ap_id) do ap_try = ActivityPub.make_user_from_ap_id(ap_id) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do true else inbox_info = URI.parse(inbox) - !Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host) + !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host) end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.AdminAPI.Config + alias Pleroma.Web.AdminAPI.ConfigView alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.CommonAPI @@ -362,6 +364,41 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end + def config_show(conn, _params) do + configs = Pleroma.Repo.all(Config) + + conn + |> put_view(ConfigView) + |> render("index.json", %{configs: configs}) + end + + def config_update(conn, %{"configs" => configs}) do + updated = + if Pleroma.Config.get([:instance, :dynamic_configuration]) do + updated = + Enum.map(configs, fn + %{"key" => key, "value" => value} -> + {:ok, config} = Config.update_or_create(%{key: key, value: value}) + config + + %{"key" => key, "delete" => "true"} -> + {:ok, _} = Config.delete(key) + nil + end) + |> Enum.reject(&is_nil(&1)) + + Pleroma.Config.TransferTask.load_and_update_env() + Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env)]) + updated + else + [] + end + + conn + |> put_view(ConfigView) + |> render("index.json", %{configs: updated}) + end + def errors(conn, {:error, :not_found}) do conn |> put_status(404) diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex @@ -0,0 +1,144 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.Config do + use Ecto.Schema + import Ecto.Changeset + alias __MODULE__ + alias Pleroma.Repo + + @type t :: %__MODULE__{} + + schema "config" do + field(:key, :string) + field(:value, :binary) + + timestamps() + end + + @spec get_by_key(String.t()) :: Config.t() | nil + def get_by_key(key), do: Repo.get_by(Config, key: key) + + @spec changeset(Config.t(), map()) :: Changeset.t() + def changeset(config, params \\ %{}) do + config + |> cast(params, [:key, :value]) + |> validate_required([:key, :value]) + |> unique_constraint(:key) + end + + @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} + def create(%{key: key, value: value}) do + %Config{} + |> changeset(%{key: key, value: transform(value)}) + |> Repo.insert() + end + + @spec update(Config.t(), map()) :: {:ok, Config} | {:error, Changeset.t()} + def update(%Config{} = config, %{value: value}) do + config + |> change(value: transform(value)) + |> Repo.update() + end + + @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} + def update_or_create(%{key: key} = params) do + with %Config{} = config <- Config.get_by_key(key) do + Config.update(config, params) + else + nil -> Config.create(params) + end + end + + @spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()} + def delete(key) do + with %Config{} = config <- Config.get_by_key(key) do + Repo.delete(config) + else + nil -> {:error, "Config with key #{key} not found"} + end + end + + @spec from_binary(binary()) :: term() + def from_binary(value), do: :erlang.binary_to_term(value) + + @spec from_binary_to_map(binary()) :: any() + def from_binary_to_map(binary) do + from_binary(binary) + |> do_convert() + end + + defp do_convert([{k, v}] = value) when is_list(value) and length(value) == 1, + do: %{k => do_convert(v)} + + defp do_convert(values) when is_list(values), do: for(val <- values, do: do_convert(val)) + + defp do_convert({k, v} = value) when is_tuple(value), + do: %{k => do_convert(v)} + + defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value), + do: value + + @spec transform(any()) :: binary() + def transform(entity) when is_map(entity) do + tuples = + for {k, v} <- entity, + into: [], + do: {if(is_atom(k), do: k, else: String.to_atom(k)), do_transform(v)} + + Enum.reject(tuples, fn {_k, v} -> is_nil(v) end) + |> Enum.sort() + |> :erlang.term_to_binary() + end + + def transform(entity) when is_list(entity) do + list = Enum.map(entity, &do_transform(&1)) + :erlang.term_to_binary(list) + end + + def transform(entity), do: :erlang.term_to_binary(entity) + + defp do_transform(%Regex{} = value) when is_map(value), do: value + + defp do_transform(value) when is_map(value) do + values = + for {key, val} <- value, + into: [], + do: {String.to_atom(key), do_transform(val)} + + Enum.sort(values) + end + + defp do_transform(value) when is_list(value) do + Enum.map(value, &do_transform(&1)) + end + + defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity) + + defp do_transform(value) when is_binary(value) do + value = String.trim(value) + + case String.length(value) do + 0 -> + nil + + _ -> + cond do + String.starts_with?(value, "Pleroma") -> + String.to_existing_atom("Elixir." <> value) + + String.starts_with?(value, ":") -> + String.replace(value, ":", "") |> String.to_existing_atom() + + String.starts_with?(value, "i:") -> + String.replace(value, "i:", "") |> String.to_integer() + + true -> + value + end + end + end + + defp do_transform(value), do: value +end diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.Web.AdminAPI.ConfigView do + use Pleroma.Web, :view + + def render("index.json", %{configs: configs}) do + %{ + configs: render_many(configs, __MODULE__, "show.json", as: :config) + } + end + + def render("show.json", %{config: config}) do + %{ + key: config.key, + value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value) + } + end +end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex @@ -91,7 +91,7 @@ defmodule Pleroma.Web.Endpoint do Plug.Session, store: :cookie, key: cookie_name, - signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]}, + signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"), http_only: true, secure: secure_cookies, extra: extra diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Web.OAuth.Token do alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token.Query - @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) @type t :: %__MODULE__{} schema "oauth_tokens" do @@ -78,7 +77,7 @@ defmodule Pleroma.Web.OAuth.Token do defp put_valid_until(changeset, attrs) do expires_in = - Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in)) + Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in())) changeset |> change(%{valid_until: expires_in}) @@ -123,4 +122,6 @@ defmodule Pleroma.Web.OAuth.Token do end def is_expired?(_), do: false + + defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) end diff --git a/lib/pleroma/web/oauth/token/response.ex b/lib/pleroma/web/oauth/token/response.ex @@ -4,15 +4,13 @@ defmodule Pleroma.Web.OAuth.Token.Response do alias Pleroma.User alias Pleroma.Web.OAuth.Token.Utils - @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) - @doc false def build(%User{} = user, token, opts \\ %{}) do %{ token_type: "Bearer", access_token: token.token, refresh_token: token.refresh_token, - expires_in: @expires_in, + expires_in: expires_in(), scope: Enum.join(token.scopes, " "), me: user.ap_id } @@ -25,8 +23,10 @@ defmodule Pleroma.Web.OAuth.Token.Response do access_token: token.token, refresh_token: token.refresh_token, created_at: Utils.format_created_at(token), - expires_in: @expires_in, + expires_in: expires_in(), scope: Enum.join(token.scopes, " ") } end + + defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex @@ -202,6 +202,9 @@ defmodule Pleroma.Web.Router do put("/statuses/:id", AdminAPIController, :status_update) delete("/statuses/:id", AdminAPIController, :status_delete) + + get("/config", AdminAPIController, :config_show) + post("/config", AdminAPIController, :config_update) end scope "/", Pleroma.Web.TwitterAPI do diff --git a/priv/repo/migrations/20190518032627_create_config.exs b/priv/repo/migrations/20190518032627_create_config.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.CreateConfig do + use Ecto.Migration + + def change do + create table(:config) do + add(:key, :string) + add(:value, :binary) + timestamps() + end + + create(unique_index(:config, :key)) + end +end diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs @@ -0,0 +1,35 @@ +defmodule Pleroma.Config.TransferTaskTest do + use Pleroma.DataCase + + setup do + dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + + Pleroma.Config.put([:instance, :dynamic_configuration], true) + + on_exit(fn -> + Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) + end) + end + + test "transfer config values from db to env" do + refute Application.get_env(:pleroma, :test_key) + Pleroma.Web.AdminAPI.Config.create(%{key: "test_key", value: [live: 2, com: 3]}) + + Pleroma.Config.TransferTask.start_link() + + assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3] + + on_exit(fn -> + Application.delete_env(:pleroma, :test_key) + end) + end + + test "non existing atom" do + Pleroma.Web.AdminAPI.Config.create(%{key: "undefined_atom_key", value: [live: 2, com: 3]}) + + assert ExUnit.CaptureLog.capture_log(fn -> + Pleroma.Config.TransferTask.start_link() + end) =~ + "updating env causes error, key: \"undefined_atom_key\", error: %ArgumentError{message: \"argument error\"}" + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex @@ -310,4 +310,17 @@ defmodule Pleroma.Factory do } } end + + def config_factory do + %Pleroma.Web.AdminAPI.Config{ + key: sequence(:key, &"some_key_#{&1}"), + value: + sequence( + :value, + fn key -> + :erlang.term_to_binary(%{another_key: "#{key}somevalue", another: "#{key}somevalue"}) + end + ) + } + end end diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs @@ -0,0 +1,54 @@ +defmodule Mix.Tasks.Pleroma.ConfigTest do + use Pleroma.DataCase + alias Pleroma.Repo + alias Pleroma.Web.AdminAPI.Config + + setup_all do + Mix.shell(Mix.Shell.Process) + temp_file = "config/temp.migrated.secret.exs" + + dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + + Pleroma.Config.put([:instance, :dynamic_configuration], true) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + Application.delete_env(:pleroma, :first_setting) + Application.delete_env(:pleroma, :second_setting) + Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) + :ok = File.rm(temp_file) + end) + + {:ok, temp_file: temp_file} + end + + test "settings are migrated to db" do + assert Repo.all(Config) == [] + + Application.put_env(:pleroma, :first_setting, key: "value", key2: [Pleroma.Repo]) + Application.put_env(:pleroma, :second_setting, key: "value2", key2: [Pleroma.Activity]) + + Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) + + first_db = Config.get_by_key("first_setting") + second_db = Config.get_by_key("second_setting") + refute Config.get_by_key("Pleroma.Repo") + + assert Config.from_binary(first_db.value) == [key: "value", key2: [Pleroma.Repo]] + assert Config.from_binary(second_db.value) == [key: "value2", key2: [Pleroma.Activity]] + end + + test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do + Config.create(%{key: "setting_first", value: [key: "value", key2: [Pleroma.Activity]]}) + Config.create(%{key: "setting_second", value: [key: "valu2", key2: [Pleroma.Repo]]}) + + Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "temp"]) + + assert Repo.all(Config) == [] + assert File.exists?(temp_file) + {:ok, file} = File.read(temp_file) + + assert file =~ "config :pleroma, setting_first:" + assert file =~ "config :pleroma, setting_second:" + end +end diff --git a/test/tasks/instance.exs b/test/tasks/instance.exs @@ -36,6 +36,8 @@ defmodule Pleroma.InstanceTest do "--dbpass", "dbpass", "--indexable", + "y", + "--db-configurable", "y" ]) end @@ -53,6 +55,7 @@ defmodule Pleroma.InstanceTest do assert generated_config =~ "database: \"dbname\"" assert generated_config =~ "username: \"dbuser\"" assert generated_config =~ "password: \"dbpass\"" + assert generated_config =~ "dynamic_configuration: true" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs @@ -1292,4 +1292,176 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert json_response(conn, :bad_request) == "Could not delete" end end + + describe "GET /api/pleroma/admin/config" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "without any settings in db", %{conn: conn} do + conn = get(conn, "/api/pleroma/admin/config") + + assert json_response(conn, 200) == %{"configs" => []} + end + + test "with settings in db", %{conn: conn} do + config1 = insert(:config) + config2 = insert(:config) + + conn = get(conn, "/api/pleroma/admin/config") + + %{ + "configs" => [ + %{ + "key" => key1, + "value" => _ + }, + %{ + "key" => key2, + "value" => _ + } + ] + } = json_response(conn, 200) + + assert key1 == config1.key + assert key2 == config2.key + end + end + + describe "POST /api/pleroma/admin/config" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + temp_file = "config/test.migrated.secret.exs" + + on_exit(fn -> + Application.delete_env(:pleroma, :key1) + Application.delete_env(:pleroma, :key2) + Application.delete_env(:pleroma, :key3) + Application.delete_env(:pleroma, :key4) + Application.delete_env(:pleroma, :keyaa1) + Application.delete_env(:pleroma, :keyaa2) + :ok = File.rm(temp_file) + end) + + dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + + Pleroma.Config.put([:instance, :dynamic_configuration], true) + + on_exit(fn -> + Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) + end) + + %{conn: assign(conn, :user, admin)} + end + + test "create new config setting in db", %{conn: conn} do + conn = + post(conn, "/api/pleroma/admin/config", %{ + configs: [ + %{key: "key1", value: "value1"}, + %{ + key: "key2", + value: %{ + "nested_1" => "nested_value1", + "nested_2" => [ + %{"nested_22" => "nested_value222"}, + %{"nested_33" => %{"nested_44" => "nested_444"}} + ] + } + }, + %{ + key: "key3", + value: [ + %{"nested_3" => ":nested_3", "nested_33" => "nested_33"}, + %{"nested_4" => ":true"} + ] + }, + %{ + key: "key4", + value: %{"nested_5" => ":upload", "endpoint" => "https://example.com"} + } + ] + }) + + assert json_response(conn, 200) == %{ + "configs" => [ + %{ + "key" => "key1", + "value" => "value1" + }, + %{ + "key" => "key2", + "value" => [ + %{"nested_1" => "nested_value1"}, + %{ + "nested_2" => [ + %{"nested_22" => "nested_value222"}, + %{"nested_33" => %{"nested_44" => "nested_444"}} + ] + } + ] + }, + %{ + "key" => "key3", + "value" => [ + [%{"nested_3" => "nested_3"}, %{"nested_33" => "nested_33"}], + %{"nested_4" => true} + ] + }, + %{ + "key" => "key4", + "value" => [%{"endpoint" => "https://example.com"}, %{"nested_5" => "upload"}] + } + ] + } + + assert Application.get_env(:pleroma, :key1) == "value1" + + assert Application.get_env(:pleroma, :key2) == [ + nested_1: "nested_value1", + nested_2: [ + [nested_22: "nested_value222"], + [nested_33: [nested_44: "nested_444"]] + ] + ] + + assert Application.get_env(:pleroma, :key3) == [ + [nested_3: :nested_3, nested_33: "nested_33"], + [nested_4: true] + ] + + assert Application.get_env(:pleroma, :key4) == [ + endpoint: "https://example.com", + nested_5: :upload + ] + end + + test "update config setting & delete", %{conn: conn} do + config1 = insert(:config, key: "keyaa1") + config2 = insert(:config, key: "keyaa2") + + conn = + post(conn, "/api/pleroma/admin/config", %{ + configs: [ + %{key: config1.key, value: "another_value"}, + %{key: config2.key, delete: "true"} + ] + }) + + assert json_response(conn, 200) == %{ + "configs" => [ + %{ + "key" => config1.key, + "value" => "another_value" + } + ] + } + + assert Application.get_env(:pleroma, :keyaa1) == "another_value" + refute Application.get_env(:pleroma, :keyaa2) + end + end end diff --git a/test/web/admin_api/config_test.exs b/test/web/admin_api/config_test.exs @@ -0,0 +1,183 @@ +defmodule Pleroma.Web.AdminAPI.ConfigTest do + use Pleroma.DataCase, async: true + import Pleroma.Factory + alias Pleroma.Web.AdminAPI.Config + + test "get_by_key/1" do + config = insert(:config) + insert(:config) + + assert config == Config.get_by_key(config.key) + end + + test "create/1" do + {:ok, config} = Config.create(%{key: "some_key", value: "some_value"}) + assert config == Config.get_by_key("some_key") + end + + test "update/1" do + config = insert(:config) + {:ok, updated} = Config.update(config, %{value: "some_value"}) + loaded = Config.get_by_key(config.key) + assert loaded == updated + end + + test "update_or_create/1" do + config = insert(:config) + key2 = "another_key" + + params = [ + %{key: key2, value: "another_value"}, + %{key: config.key, value: "new_value"} + ] + + assert Repo.all(Config) |> length() == 1 + + Enum.each(params, &Config.update_or_create(&1)) + + assert Repo.all(Config) |> length() == 2 + + config1 = Config.get_by_key(config.key) + config2 = Config.get_by_key(key2) + + assert config1.value == Config.transform("new_value") + assert config2.value == Config.transform("another_value") + end + + test "delete/1" do + config = insert(:config) + {:ok, _} = Config.delete(config.key) + refute Config.get_by_key(config.key) + end + + describe "transform/1" do + test "string" do + binary = Config.transform("value as string") + assert binary == :erlang.term_to_binary("value as string") + assert Config.from_binary(binary) == "value as string" + end + + test "list of modules" do + binary = Config.transform(["Pleroma.Repo", "Pleroma.Activity"]) + assert binary == :erlang.term_to_binary([Pleroma.Repo, Pleroma.Activity]) + assert Config.from_binary(binary) == [Pleroma.Repo, Pleroma.Activity] + end + + test "list of strings" do + binary = Config.transform(["string1", "string2"]) + assert binary == :erlang.term_to_binary(["string1", "string2"]) + assert Config.from_binary(binary) == ["string1", "string2"] + end + + test "map" do + binary = + Config.transform(%{ + "types" => "Pleroma.PostgresTypes", + "telemetry_event" => ["Pleroma.Repo.Instrumenter"], + "migration_lock" => "" + }) + + assert binary == + :erlang.term_to_binary( + telemetry_event: [Pleroma.Repo.Instrumenter], + types: Pleroma.PostgresTypes + ) + + assert Config.from_binary(binary) == [ + telemetry_event: [Pleroma.Repo.Instrumenter], + types: Pleroma.PostgresTypes + ] + end + + test "complex map with nested integers, lists and atoms" do + binary = + Config.transform(%{ + "uploader" => "Pleroma.Uploaders.Local", + "filters" => ["Pleroma.Upload.Filter.Dedupe"], + "link_name" => ":true", + "proxy_remote" => ":false", + "proxy_opts" => %{ + "redirect_on_failure" => ":false", + "max_body_length" => "i:1048576", + "http" => %{ + "follow_redirect" => ":true", + "pool" => ":upload" + } + } + }) + + assert binary == + :erlang.term_to_binary( + filters: [Pleroma.Upload.Filter.Dedupe], + link_name: true, + proxy_opts: [ + http: [ + follow_redirect: true, + pool: :upload + ], + max_body_length: 1_048_576, + redirect_on_failure: false + ], + proxy_remote: false, + uploader: Pleroma.Uploaders.Local + ) + + assert Config.from_binary(binary) == + [ + filters: [Pleroma.Upload.Filter.Dedupe], + link_name: true, + proxy_opts: [ + http: [ + follow_redirect: true, + pool: :upload + ], + max_body_length: 1_048_576, + redirect_on_failure: false + ], + proxy_remote: false, + uploader: Pleroma.Uploaders.Local + ] + end + + test "keyword" do + binary = + Config.transform(%{ + "level" => ":warn", + "meta" => [":all"], + "webhook_url" => "https://hooks.slack.com/services/YOUR-KEY-HERE" + }) + + assert binary == + :erlang.term_to_binary( + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE" + ) + + assert Config.from_binary(binary) == [ + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE" + ] + end + + test "complex map with sigil" do + binary = + Config.transform(%{ + federated_timeline_removal: [], + reject: [~r/comp[lL][aA][iI][nN]er/], + replace: [] + }) + + assert binary == + :erlang.term_to_binary( + federated_timeline_removal: [], + reject: [~r/comp[lL][aA][iI][nN]er/], + replace: [] + ) + + assert Config.from_binary(binary) == + [federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []] + end + end +end