commit: 35ac549a99f325d2e73c03d7c2951f3d6570b7b4
parent: 2a744cdff8a94d92683ab1a16f89bb51112006f8
Author: lambda <pleromagit@rogerbraun.net>
Date: Wed, 20 Dec 2017 14:22:24 +0000
Merge branch 'media-proxy' into 'develop'
Media proxy
See merge request pleroma/pleroma!34
Diffstat:
10 files changed, 152 insertions(+), 11 deletions(-)
diff --git a/config/config.exs b/config/config.exs
@@ -47,6 +47,11 @@ config :pleroma, :instance,
limit: 5000,
registrations_open: true
+config :pleroma, :media_proxy,
+ enabled: false,
+ redirect_on_failure: true
+ #base_url: "https://cache.pleroma.social"
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env}.exs"
diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
@@ -1,3 +1,6 @@
+proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
+ inactive=720m use_temp_path=off;
+
server {
listen 80;
server_name example.tld;
@@ -24,6 +27,13 @@ server {
proxy_set_header Connection "upgrade";
proxy_pass http://localhost:4000;
}
+
+ location /proxy {
+ proxy_cache pleroma_media_cache;
+ proxy_cache_lock on;
+ proxy_pass http://localhost:4000;
+ }
+
include snippets/well-known.conf;
-}-
\ No newline at end of file
+}
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
@@ -1,5 +1,6 @@
defmodule Pleroma.Formatter do
alias Pleroma.User
+ alias Pleroma.Web.MediaProxy
@link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
def linkify(text) do
@@ -125,7 +126,7 @@ defmodule Pleroma.Formatter do
end
Enum.reduce(all_emoji, text, fn ({emoji, file}, text) ->
- String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{file}' />")
+ String.replace(text, ":#{emoji}:", "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />")
end)
end
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
alias Pleroma.User
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.CommonAPI.Utils
+ alias Pleroma.Web.MediaProxy
defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href
defp image_url(_), do: nil
@@ -12,10 +13,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
end
def render("account.json", %{user: user}) do
- image = User.avatar_url(user)
+ image = User.avatar_url(user) |> MediaProxy.url()
user_info = User.user_info(user)
- header = image_url(user.info["banner"]) || "https://placehold.it/700x335"
+ header = (image_url(user.info["banner"]) || "https://placehold.it/700x335") |> MediaProxy.url()
%{
id: to_string(user.id),
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -3,6 +3,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
alias Pleroma.{User, Activity}
alias Pleroma.Web.CommonAPI.Utils
+ alias Pleroma.Web.MediaProxy
def render("index.json", opts) do
render_many(opts.activities, StatusView, "status.json", opts)
@@ -121,9 +122,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
%{
id: to_string(attachment["id"] || hash_id),
- url: href,
+ url: MediaProxy.url(href),
remote_url: href,
- preview_url: href,
+ preview_url: MediaProxy.url(href),
text_url: href,
type: type
}
diff --git a/lib/pleroma/web/media_proxy/controller.ex b/lib/pleroma/web/media_proxy/controller.ex
@@ -0,0 +1,83 @@
+defmodule Pleroma.Web.MediaProxy.MediaProxyController do
+ use Pleroma.Web, :controller
+ require Logger
+
+ @max_body_length 25 * 1048576
+
+ @cache_control %{
+ default: "public, max-age=1209600",
+ error: "public, must-revalidate, max-age=160",
+ }
+
+ def remote(conn, %{"sig" => sig, "url" => url}) do
+ config = Application.get_env(:pleroma, :media_proxy, [])
+ with \
+ true <- Keyword.get(config, :enabled, false),
+ {:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
+ url = URI.encode(url),
+ {:ok, content_type, body} <- proxy_request(url)
+ do
+ conn
+ |> put_resp_content_type(content_type)
+ |> set_cache_header(:default)
+ |> send_resp(200, body)
+ else
+ false -> send_error(conn, 404)
+ {:error, :invalid_signature} -> send_error(conn, 403)
+ {:error, {:http, _, url}} -> redirect_or_error(conn, url, Keyword.get(config, :redirect_on_failure, true))
+ end
+ end
+
+ defp proxy_request(link) do
+ headers = [{"user-agent", "Pleroma/MediaProxy; #{Pleroma.Web.base_url()} <#{Application.get_env(:pleroma, :instance)[:email]}>"}]
+ options = [:insecure, {:follow_redirect, true}]
+ with \
+ {:ok, 200, headers, client} <- :hackney.request(:get, link, headers, "", options),
+ headers = Enum.into(headers, Map.new),
+ {:ok, body} <- proxy_request_body(client),
+ content_type <- proxy_request_content_type(headers, body)
+ do
+ {:ok, content_type, body}
+ else
+ {:ok, status, _, _} ->
+ Logger.warn "MediaProxy: request failed, status #{status}, link: #{link}"
+ {:error, {:http, :bad_status, link}}
+ {:error, error} ->
+ Logger.warn "MediaProxy: request failed, error #{inspect error}, link: #{link}"
+ {:error, {:http, error, link}}
+ end
+ end
+
+ defp set_cache_header(conn, key) do
+ Plug.Conn.put_resp_header(conn, "cache-control", @cache_control[key])
+ end
+
+ defp redirect_or_error(conn, url, true), do: redirect(conn, external: url)
+ defp redirect_or_error(conn, url, _), do: send_error(conn, 502, "Media proxy error: " <> url)
+
+ defp send_error(conn, code, body \\ "") do
+ conn
+ |> set_cache_header(:error)
+ |> send_resp(code, body)
+ end
+
+ defp proxy_request_body(client), do: proxy_request_body(client, <<>>)
+ defp proxy_request_body(client, body) when byte_size(body) < @max_body_length do
+ case :hackney.stream_body(client) do
+ {:ok, data} -> proxy_request_body(client, <<body :: binary, data :: binary>>)
+ :done -> {:ok, body}
+ {:error, reason} -> {:error, reason}
+ end
+ end
+ defp proxy_request_body(client, _) do
+ :hackney.close(client)
+ {:error, :body_too_large}
+ end
+
+ # TODO: the body is passed here as well because some hosts do not provide a content-type.
+ # At some point we may want to use magic numbers to discover the content-type and reply a proper one.
+ defp proxy_request_content_type(headers, _body) do
+ headers["Content-Type"] || headers["content-type"] || "image/jpeg"
+ end
+
+end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -0,0 +1,32 @@
+defmodule Pleroma.Web.MediaProxy do
+ @base64_opts [padding: false]
+
+ def url(nil), do: nil
+
+ def url(url = "/" <> _), do: url
+
+ def url(url) do
+ config = Application.get_env(:pleroma, :media_proxy, [])
+ if !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url) do
+ url
+ else
+ secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+ base64 = Base.url_encode64(url, @base64_opts)
+ sig = :crypto.hmac(:sha, secret, base64)
+ sig64 = sig |> Base.url_encode64(@base64_opts)
+ Keyword.get(config, :base_url, Pleroma.Web.base_url) <> "/proxy/#{sig64}/#{base64}"
+ end
+ end
+
+ def decode_url(sig, url) do
+ secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+ sig = Base.url_decode64!(sig, @base64_opts)
+ local_sig = :crypto.hmac(:sha, secret, url)
+ if local_sig == sig do
+ {:ok, Base.url_decode64!(url, @base64_opts)}
+ else
+ {:error, :invalid_signature}
+ end
+ end
+
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
@@ -238,6 +238,14 @@ defmodule Pleroma.Web.Router do
delete "/auth/sign_out", MastodonAPIController, :logout
end
+ pipeline :remote_media do
+ plug :accepts, ["html"]
+ end
+ scope "/proxy/", Pleroma.Web.MediaProxy do
+ pipe_through :remote_media
+ get "/:sig/:url", MediaProxyController, :remote
+ end
+
scope "/", Fallback do
get "/*path", RedirectController, :redirector
end
diff --git a/lib/pleroma/web/twitter_api/representers/object_representer.ex b/lib/pleroma/web/twitter_api/representers/object_representer.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter do
data = object.data
url = List.first(data["url"])
%{
- url: url["href"],
+ url: url["href"] |> Pleroma.Web.MediaProxy.url(),
mimetype: url["mediaType"],
id: data["uuid"],
oembed: false
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -2,6 +2,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
use Pleroma.Web, :view
alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils
+ alias Pleroma.Web.MediaProxy
def render("show.json", %{user: user = %User{}} = assigns) do
render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@@ -12,7 +13,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
end
def render("user.json", %{user: user = %User{}} = assigns) do
- image = User.avatar_url(user)
+ image = User.avatar_url(user) |> MediaProxy.url()
{following, follows_you, statusnet_blocking} = if assigns[:for] do
{
User.following?(assigns[:for], user),
@@ -44,8 +45,8 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
"screen_name" => user.nickname,
"statuses_count" => user_info[:note_count],
"statusnet_profile_url" => user.ap_id,
- "cover_photo" => image_url(user.info["banner"]),
- "background_image" => image_url(user.info["background"])
+ "cover_photo" => image_url(user.info["banner"]) |> MediaProxy.url(),
+ "background_image" => image_url(user.info["background"]) |> MediaProxy.url(),
}
if assigns[:token] do
