commit: fd48bd80eb7eab563c25586b1deba2ed42c02c5a parent: 472e7b796cfeb1445ee1572df414531655b050ce Author: rinpatch <rinpatch@sdf.org> Date: Mon, 23 Sep 2019 18:20:08 +0000 Merge branch 'better-emoji-packs' into 'develop' Shareable emoji packs Closes #833 and #1096 See merge request pleroma/pleroma!1551Diffstat:
19 files changed, 1192 insertions(+), 15 deletions(-)
diff --git a/config/config.exs b/config/config.exs @@ -122,7 +122,8 @@ config :pleroma, :emoji, # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` Custom: ["/emoji/*.png", "/emoji/**/*.png"] ], - default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json" + default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json", + shared_pack_cache_seconds_per_file: 60 config :pleroma, :uri_schemes, valid_schemes: [ diff --git a/config/description.exs b/config/description.exs @@ -2256,6 +2256,14 @@ config :pleroma, :config_description, [ "Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download." <> " Currently only one manifest can be added (no arrays)", suggestions: ["https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"] + }, + %{ + key: :shared_pack_cache_seconds_per_file, + type: :integer, + descpiption: + "When an emoji pack is shared, the archive is created and cached in memory" <> + " for this amount of seconds multiplied by the number of files.", + suggestions: [60] } ] }, diff --git a/config/test.exs b/config/test.exs @@ -30,7 +30,8 @@ config :pleroma, :instance, notify_email: "noreply@example.com", skip_thread_containment: false, federating: false, - external_user_synchronization: false + external_user_synchronization: false, + static_dir: "test/instance_static/" config :pleroma, :activitypub, sign_object_fetches: false diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md @@ -733,3 +733,10 @@ Compile time settings (need instance reboot): } ] ``` + +## `POST /api/pleroma/admin/reload_emoji` +### Reload the instance's custom emoji +* Method `POST` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md @@ -365,3 +365,68 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) + +## `GET /api/pleroma/emoji/packs` +### Lists the custom emoji packs on the server +* Method `GET` +* Authentication: not required +* Params: None +* Response: JSON, "ok" and 200 status and the JSON hashmap of "pack name" to "pack contents" + +## `PUT /api/pleroma/emoji/packs/:name` +### Creates an empty custom emoji pack +* Method `PUT` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists + +## `DELETE /api/pleroma/emoji/packs/:name` +### Delete a custom emoji pack +* Method `DELETE` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack + +## `POST /api/pleroma/emoji/packs/:name/update_file` +### Update a file in a custom emoji pack +* Method `POST` +* Authentication: required +* Params: + * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + * if the `action` is `update`, changes emoji shortcode + (from `shortcode` to `new_shortcode` or moves the file (from the current filename to `new_filename`) + * if the `action` is `remove`, removes the emoji named `shortcode` and it's associated file +* Response: JSON, updated "files" section of the pack and 200 status, 409 if the trying to use a shortcode + that is already taken, 400 if there was an error with the shortcode, filename or file (additional info + in the "error" part of the response JSON) + +## `POST /api/pleroma/emoji/packs/:name/update_metadata` +### Updates (replaces) pack metadata +* Method `POST` +* Authentication: required +* Params: + * `new_data`: new metadata to replace the old one +* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a + problem with the new metadata (the error is specified in the "error" part of the response JSON) + +## `POST /api/pleroma/emoji/packs/download_from` +### Requests the instance to download the pack from another instance +* Method `POST` +* Authentication: required +* Params: + * `instance_address`: the address of the instance to download from + * `pack_name`: the pack to download from that instance +* Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were + errors downloading the pack + +## `GET /api/pleroma/emoji/packs/:name/download_shared` +### Requests a local pack from the instance +* Method `GET` +* Authentication: not required +* Params: None +* Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared, + 404 if the pack does not exist diff --git a/docs/config.md b/docs/config.md @@ -707,6 +707,8 @@ Configure OAuth 2 provider capabilities: * `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). +* `shared_pack_cache_seconds_per_file`: When an emoji pack is shared, the archive is created and cached in + memory for this amount of seconds multiplied by the number of files. ## Database options diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex @@ -102,10 +102,14 @@ defmodule Pleroma.Application do build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), - build_cachex("web_resp", limit: 2500) + build_cachex("web_resp", limit: 2500), + build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10) ] end + defp emoji_packs_expiration, + do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) + defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex @@ -122,6 +122,9 @@ defmodule Pleroma.Emoji do fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end ) + # Clear out old emojis + :ets.delete_all_objects(@ets) + true = :ets.insert(@ets, emojis) end @@ -143,23 +146,38 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - emoji_txt = Path.join(pack_dir, "emoji.txt") + pack_file = Path.join(pack_dir, "pack.json") + + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) + contents["files"] + |> Enum.map(fn {name, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + {name, filename, pack_name} + end) else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" - ) + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -599,6 +599,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{configs: updated}) end + def reload_emoji(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", + "shareable_emoji_packs", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -0,0 +1,575 @@ +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do + use Pleroma.Web, :controller + + require Logger + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ + def list_packs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) + + json(conn, pack_infos) + end + end + + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end + + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") + + {pack_name, Jason.decode!(File.read!(pack_file))} + end + + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] && + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp create_archive_and_cache(name, pack, pack_dir, md5) do + files = + ['pack.json'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Created an archive for the '#{name}' emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) + + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result + + _ -> + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) + end + end + + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file = Path.join(pack_dir, "pack.json") + + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") + else + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) + + {:exists?, _} -> + conn + |> put_status(:not_found) + |> json(%{error: "Pack #{name} does not exist"}) + end + end + + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + shareable_packs_available = + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) + + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end + + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") + + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) + + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + end + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + if not File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}) + ) + + conn |> json("ok") + else + conn + |> put_status(:conflict) + |> json(%{error: "A pack named \"#{name}\" already exists"}) + end + end + + @doc """ + Deletes the pack `name` and all it's files. + """ + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> json("ok") + + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) + end + end + + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) + end + end + + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end + + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + json(conn, new_data) + end + + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ + + # Add + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) + end + end + + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + end + end + + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ + def import_from_fs(conn, _params) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Error accessing emoji pack directory"}) + end + end + + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) + + dir + end + + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex @@ -205,6 +205,29 @@ defmodule Pleroma.Web.Router do get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/moderation_log", AdminAPIController, :list_log) + + post("/reload_emoji", AdminAPIController, :reload_emoji) + end + + scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do + scope "/packs" do + # Modifying packs + pipe_through([:admin_api, :oauth_write]) + + post("/import_from_fs", EmojiAPIController, :import_from_fs) + + post("/:pack_name/update_file", EmojiAPIController, :update_file) + post("/:pack_name/update_metadata", EmojiAPIController, :update_metadata) + put("/:name", EmojiAPIController, :create) + delete("/:name", EmojiAPIController, :delete) + post("/download_from", EmojiAPIController, :download_from) + end + + scope "/packs" do + # Pack info / downloading + get("/", EmojiAPIController, :list_packs) + get("/:name/download_shared/", EmojiAPIController, :download_shared) + end end scope "/", Pleroma.Web.TwitterAPI do diff --git a/test/instance_static/emoji/test_pack/blank.png b/test/instance_static/emoji/test_pack/blank.png Binary files differ. diff --git a/test/instance_static/emoji/test_pack/pack.json b/test/instance_static/emoji/test_pack/pack.json @@ -0,0 +1,13 @@ +{ + "pack": { + "license": "Test license", + "homepage": "https://pleroma.social", + "description": "Test description", + + "share-files": true + }, + + "files": { + "blank": "blank.png" + } +} diff --git a/test/instance_static/emoji/test_pack_for_import/blank.png b/test/instance_static/emoji/test_pack_for_import/blank.png Binary files differ. diff --git a/test/instance_static/emoji/test_pack_nonshared/nonshared.zip b/test/instance_static/emoji/test_pack_nonshared/nonshared.zip Binary files differ. diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.json b/test/instance_static/emoji/test_pack_nonshared/pack.json @@ -0,0 +1,16 @@ +{ + "pack": { + "license": "Test license", + "homepage": "https://pleroma.social", + "description": "Test description", + + "fallback-src": "https://nonshared-pack", + "fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF", + + "share-files": false + }, + + "files": { + "blank": "blank.png" + } +} diff --git a/test/web/pleroma_api/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs @@ -0,0 +1,437 @@ +defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do + use Pleroma.Web.ConnCase + + import Tesla.Mock + + import Pleroma.Factory + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + test "shared & non-shared pack information in list_packs is ok" do + conn = build_conn() + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + assert Map.has_key?(resp, "test_pack") + + pack = resp["test_pack"] + + assert Map.has_key?(pack["pack"], "download-sha256") + assert pack["pack"]["can-download"] + + assert pack["files"] == %{"blank" => "blank.png"} + + # Non-shared pack + + assert Map.has_key?(resp, "test_pack_nonshared") + + pack = resp["test_pack_nonshared"] + + refute pack["pack"]["shared"] + refute pack["pack"]["can-download"] + end + + test "downloading a shared pack from download_shared" do + conn = build_conn() + + resp = + conn + |> get(emoji_api_path(conn, :download_shared, "test_pack")) + |> response(200) + + {:ok, arch} = :zip.unzip(resp, [:memory]) + + assert Enum.find(arch, fn {n, _} -> n == 'pack.json' end) + assert Enum.find(arch, fn {n, _} -> n == 'blank.png' end) + end + + test "downloading shared & unshared packs from another instance via download_from, deleting them" do + on_exit(fn -> + File.rm_rf!("#{@emoji_dir_path}/test_pack2") + File.rm_rf!("#{@emoji_dir_path}/test_pack_nonshared2") + end) + + mock(fn + %{method: :get, url: "https://old-instance/.well-known/nodeinfo"} -> + json([%{href: "https://old-instance/nodeinfo/2.1.json"}]) + + %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> + json(%{metadata: %{features: []}}) + + %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> + json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + + %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> + json(%{metadata: %{features: ["shareable_emoji_packs"]}}) + + %{ + method: :get, + url: "https://example.com/api/pleroma/emoji/packs/list" + } -> + conn = build_conn() + + conn + |> get(emoji_api_path(conn, :list_packs)) + |> json_response(200) + |> json() + + %{ + method: :get, + url: "https://example.com/api/pleroma/emoji/packs/download_shared/test_pack" + } -> + conn = build_conn() + + conn + |> get(emoji_api_path(conn, :download_shared, "test_pack")) + |> response(200) + |> text() + + %{ + method: :get, + url: "https://nonshared-pack" + } -> + text(File.read!("#{@emoji_dir_path}/test_pack_nonshared/nonshared.zip")) + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() |> assign(:user, admin) + + assert (conn + |> put_req_header("content-type", "application/json") + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://old-instance", + pack_name: "test_pack", + as: "test_pack2" + } + |> Jason.encode!() + ) + |> json_response(500))["error"] =~ "does not support" + + assert conn + |> put_req_header("content-type", "application/json") + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://example.com", + pack_name: "test_pack", + as: "test_pack2" + } + |> Jason.encode!() + ) + |> json_response(200) == "ok" + + assert File.exists?("#{@emoji_dir_path}/test_pack2/pack.json") + assert File.exists?("#{@emoji_dir_path}/test_pack2/blank.png") + + assert conn + |> delete(emoji_api_path(conn, :delete, "test_pack2")) + |> json_response(200) == "ok" + + refute File.exists?("#{@emoji_dir_path}/test_pack2") + + # non-shared, downloaded from the fallback URL + + conn = build_conn() |> assign(:user, admin) + + assert conn + |> put_req_header("content-type", "application/json") + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://example.com", + pack_name: "test_pack_nonshared", + as: "test_pack_nonshared2" + } + |> Jason.encode!() + ) + |> json_response(200) == "ok" + + assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/pack.json") + assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/blank.png") + + assert conn + |> delete(emoji_api_path(conn, :delete, "test_pack_nonshared2")) + |> json_response(200) == "ok" + + refute File.exists?("#{@emoji_dir_path}/test_pack_nonshared2") + end + + describe "updating pack metadata" do + setup do + pack_file = "#{@emoji_dir_path}/test_pack/pack.json" + original_content = File.read!(pack_file) + + on_exit(fn -> + File.write!(pack_file, original_content) + end) + + {:ok, + admin: insert(:user, info: %{is_admin: true}), + pack_file: pack_file, + new_data: %{ + "license" => "Test license changed", + "homepage" => "https://pleroma.social", + "description" => "Test description", + "share-files" => false + }} + end + + test "for a pack without a fallback source", ctx do + conn = build_conn() + + assert conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => ctx[:new_data] + } + ) + |> json_response(200) == ctx[:new_data] + + assert Jason.decode!(File.read!(ctx[:pack_file]))["pack"] == ctx[:new_data] + end + + test "for a pack with a fallback source", ctx do + mock(fn + %{ + method: :get, + url: "https://nonshared-pack" + } -> + text(File.read!("#{@emoji_dir_path}/test_pack_nonshared/nonshared.zip")) + end) + + new_data = Map.put(ctx[:new_data], "fallback-src", "https://nonshared-pack") + + new_data_with_sha = + Map.put( + new_data, + "fallback-src-sha256", + "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF" + ) + + conn = build_conn() + + assert conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => new_data + } + ) + |> json_response(200) == new_data_with_sha + + assert Jason.decode!(File.read!(ctx[:pack_file]))["pack"] == new_data_with_sha + end + + test "when the fallback source doesn't have all the files", ctx do + mock(fn + %{ + method: :get, + url: "https://nonshared-pack" + } -> + {:ok, {'empty.zip', empty_arch}} = :zip.zip('empty.zip', [], [:memory]) + text(empty_arch) + end) + + new_data = Map.put(ctx[:new_data], "fallback-src", "https://nonshared-pack") + + conn = build_conn() + + assert (conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => new_data + } + ) + |> json_response(:bad_request))["error"] =~ "does not have all" + end + end + + test "updating pack files" do + pack_file = "#{@emoji_dir_path}/test_pack/pack.json" + original_content = File.read!(pack_file) + + on_exit(fn -> + File.write!(pack_file, original_content) + + File.rm_rf!("#{@emoji_dir_path}/test_pack/blank_url.png") + File.rm_rf!("#{@emoji_dir_path}/test_pack/dir") + File.rm_rf!("#{@emoji_dir_path}/test_pack/dir_2") + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() + + same_name = %{ + "action" => "add", + "shortcode" => "blank", + "filename" => "dir/blank.png", + "file" => %Plug.Upload{ + filename: "blank.png", + path: "#{@emoji_dir_path}/test_pack/blank.png" + } + } + + different_name = %{same_name | "shortcode" => "blank_2"} + + conn = conn |> assign(:user, admin) + + assert (conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), same_name) + |> json_response(:conflict))["error"] =~ "already exists" + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), different_name) + |> json_response(200) == %{"blank" => "blank.png", "blank_2" => "dir/blank.png"} + + assert File.exists?("#{@emoji_dir_path}/test_pack/dir/blank.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "update", + "shortcode" => "blank_2", + "new_shortcode" => "blank_3", + "new_filename" => "dir_2/blank_3.png" + }) + |> json_response(200) == %{"blank" => "blank.png", "blank_3" => "dir_2/blank_3.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/dir/") + assert File.exists?("#{@emoji_dir_path}/test_pack/dir_2/blank_3.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "remove", + "shortcode" => "blank_3" + }) + |> json_response(200) == %{"blank" => "blank.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/dir_2/") + + mock(fn + %{ + method: :get, + url: "https://test-blank/blank_url.png" + } -> + text(File.read!("#{@emoji_dir_path}/test_pack/blank.png")) + end) + + # The name should be inferred from the URL ending + from_url = %{ + "action" => "add", + "shortcode" => "blank_url", + "file" => "https://test-blank/blank_url.png" + } + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), from_url) + |> json_response(200) == %{ + "blank" => "blank.png", + "blank_url" => "blank_url.png" + } + + assert File.exists?("#{@emoji_dir_path}/test_pack/blank_url.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "remove", + "shortcode" => "blank_url" + }) + |> json_response(200) == %{"blank" => "blank.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/blank_url.png") + end + + test "creating and deleting a pack" do + on_exit(fn -> + File.rm_rf!("#{@emoji_dir_path}/test_created") + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() |> assign(:user, admin) + + assert conn + |> put_req_header("content-type", "application/json") + |> put( + emoji_api_path( + conn, + :create, + "test_created" + ) + ) + |> json_response(200) == "ok" + + assert File.exists?("#{@emoji_dir_path}/test_created/pack.json") + + assert Jason.decode!(File.read!("#{@emoji_dir_path}/test_created/pack.json")) == %{ + "pack" => %{}, + "files" => %{} + } + + assert conn + |> delete(emoji_api_path(conn, :delete, "test_created")) + |> json_response(200) == "ok" + + refute File.exists?("#{@emoji_dir_path}/test_created/pack.json") + end + + test "filesystem import" do + on_exit(fn -> + File.rm!("#{@emoji_dir_path}/test_pack_for_import/emoji.txt") + File.rm!("#{@emoji_dir_path}/test_pack_for_import/pack.json") + end) + + conn = build_conn() + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + refute Map.has_key?(resp, "test_pack_for_import") + + admin = insert(:user, info: %{is_admin: true}) + + assert conn + |> assign(:user, admin) + |> post(emoji_api_path(conn, :import_from_fs)) + |> json_response(200) == ["test_pack_for_import"] + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"} + + File.rm!("#{@emoji_dir_path}/test_pack_for_import/pack.json") + refute File.exists?("#{@emoji_dir_path}/test_pack_for_import/pack.json") + + emoji_txt_content = "blank, blank.png, Fun\n\nblank2, blank.png" + + File.write!("#{@emoji_dir_path}/test_pack_for_import/emoji.txt", emoji_txt_content) + + assert conn + |> assign(:user, admin) + |> post(emoji_api_path(conn, :import_from_fs)) + |> json_response(200) == ["test_pack_for_import"] + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + assert resp["test_pack_for_import"]["files"] == %{ + "blank" => "blank.png", + "blank2" => "blank.png" + } + end +end