commit: 65e8d47cfb74e76bf966a0340fd5ced184c74fec
parent: 1a70d41f8e7ce0eb30242d58077b69816e443c10
Author: kaniini <nenolod@gmail.com>
Date: Thu, 30 Aug 2018 23:05:01 +0000
Merge branch 'backendhack' into 'develop'
Flexible Storage Backends
See merge request pleroma/pleroma!304
Diffstat:
10 files changed, 227 insertions(+), 109 deletions(-)
diff --git a/config/config.exs b/config/config.exs
@@ -11,10 +11,12 @@ config :pleroma, ecto_repos: [Pleroma.Repo]
config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
config :pleroma, Pleroma.Upload,
- uploads: "uploads",
- strip_exif: false,
- use_s3: false,
- s3_bucket: nil
+ uploader: Pleroma.Uploaders.Local,
+ strip_exif: false
+
+config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
+
+config :pleroma, Pleroma.Uploaders.S3, s3_bucket: nil
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex
@@ -29,8 +29,7 @@ config :pleroma, Pleroma.Repo,
# The public S3 endpoint is different depending on region and provider,
# consult your S3 provider's documentation for details on what to use.
#
-# config :pleroma, Pleroma.Upload,
-# use_s3: true,
+# config :pleroma, Pleroma.Uploaders.S3,
# bucket: "some-bucket",
# public_endpoint: "https://s3.amazonaws.com"
#
@@ -44,3 +43,21 @@ config :pleroma, Pleroma.Repo,
# For using third-party S3 clones like wasabi, also do:
# config :ex_aws, :s3,
# host: "s3.wasabisys.com"
+
+
+# Configure Openstack Swift support if desired.
+#
+# Many openstack deployments are different, so config is left very open with
+# no assumptions made on which provider you're using. This should allow very
+# wide support without needing separate handlers for OVH, Rackspace, etc.
+#
+# config :pleroma, Pleroma.Uploaders.Swift,
+# container: "some-container",
+# username: "api-username-yyyy",
+# password: "api-key-xxxx",
+# tenant_id: "<openstack-project/tenant-id>",
+# auth_url: "https://keystone-endpoint.provider.com",
+# storage_url: "https://swift-endpoint.prodider.com/v1/AUTH_<tenant>/<container>",
+# object_url: "https://cdn-endpoint.provider.com/<container>"
+#
+
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
@@ -1,34 +1,19 @@
defmodule Pleroma.Upload do
alias Ecto.UUID
- alias Pleroma.Web
- def store(%Plug.Upload{} = file, should_dedupe) do
- settings = Application.get_env(:pleroma, Pleroma.Upload)
- use_s3 = Keyword.fetch!(settings, :use_s3)
+ @storage_backend Application.get_env(:pleroma, Pleroma.Upload)
+ |> Keyword.fetch!(:uploader)
+ def store(%Plug.Upload{} = file, should_dedupe) do
content_type = get_content_type(file.path)
+
uuid = get_uuid(file, should_dedupe)
name = get_name(file, uuid, content_type, should_dedupe)
- upload_folder = get_upload_path(uuid, should_dedupe)
- url_path = get_url(name, uuid, should_dedupe)
strip_exif_data(content_type, file.path)
- File.mkdir_p!(upload_folder)
- result_file = Path.join(upload_folder, name)
-
- if File.exists?(result_file) do
- File.rm!(file.path)
- else
- File.cp!(file.path, result_file)
- end
-
- url_path =
- if use_s3 do
- put_s3_file(name, uuid, result_file, content_type)
- else
- url_path
- end
+ {:ok, url_path} =
+ @storage_backend.put_file(name, uuid, file.path, content_type, should_dedupe)
%{
"type" => "Document",
@@ -43,22 +28,16 @@ defmodule Pleroma.Upload do
}
end
- # XXX: does this code actually work? i am skeptical. --kaniini
def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
- settings = Application.get_env(:pleroma, Pleroma.Upload)
- use_s3 = Keyword.fetch!(settings, :use_s3)
-
parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
data = Base.decode64!(parsed["data"], ignore: :whitespace)
- uuid = UUID.generate()
- uuidpath = Path.join(upload_path(), uuid)
- uuid = UUID.generate()
- File.mkdir_p!(upload_path())
+ tmp_path = tempfile_for_image(data)
- File.write!(uuidpath, data)
+ uuid = UUID.generate()
- content_type = get_content_type(uuidpath)
+ content_type = get_content_type(tmp_path)
+ strip_exif_data(content_type, tmp_path)
name =
create_name(
@@ -67,30 +46,7 @@ defmodule Pleroma.Upload do
content_type
)
- upload_folder = get_upload_path(uuid, should_dedupe)
- url_path = get_url(name, uuid, should_dedupe)
-
- File.mkdir_p!(upload_folder)
- result_file = Path.join(upload_folder, name)
-
- if should_dedupe do
- if !File.exists?(result_file) do
- File.rename(uuidpath, result_file)
- else
- File.rm!(uuidpath)
- end
- else
- File.rename(uuidpath, result_file)
- end
-
- strip_exif_data(content_type, result_file)
-
- url_path =
- if use_s3 do
- put_s3_file(name, uuid, result_file, content_type)
- else
- url_path
- end
+ {:ok, url_path} = @storage_backend.put_file(name, uuid, tmp_path, content_type, should_dedupe)
%{
"type" => "Image",
@@ -105,21 +61,28 @@ defmodule Pleroma.Upload do
}
end
+ @doc """
+ Creates a tempfile using the Plug.Upload Genserver which cleans them up
+ automatically.
+ """
+ def tempfile_for_image(data) do
+ {:ok, tmp_path} = Plug.Upload.random_file("profile_pics")
+ {:ok, tmp_file} = File.open(tmp_path, [:write, :raw, :binary])
+ IO.binwrite(tmp_file, data)
+
+ tmp_path
+ end
+
def strip_exif_data(content_type, file) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
do_strip = Keyword.fetch!(settings, :strip_exif)
- [filetype, ext] = String.split(content_type, "/")
+ [filetype, _ext] = String.split(content_type, "/")
if filetype == "image" and do_strip == true do
Mogrify.open(file) |> Mogrify.custom("strip") |> Mogrify.save(in_place: true)
end
end
- def upload_path do
- settings = Application.get_env(:pleroma, Pleroma.Upload)
- Keyword.fetch!(settings, :uploads)
- end
-
defp create_name(uuid, ext, type) do
case type do
"application/octet-stream" ->
@@ -163,26 +126,6 @@ defmodule Pleroma.Upload do
end
end
- defp get_upload_path(uuid, should_dedupe) do
- if should_dedupe do
- upload_path()
- else
- Path.join(upload_path(), uuid)
- end
- end
-
- defp get_url(name, uuid, should_dedupe) do
- if should_dedupe do
- url_for(:cow_uri.urlencode(name))
- else
- url_for(Path.join(uuid, :cow_uri.urlencode(name)))
- end
- end
-
- defp url_for(file) do
- "#{Web.base_url()}/media/#{file}"
- end
-
def get_content_type(file) do
match =
File.open(file, [:read], fn f ->
@@ -224,25 +167,4 @@ defmodule Pleroma.Upload do
_e -> "application/octet-stream"
end
end
-
- defp put_s3_file(name, uuid, path, content_type) do
- settings = Application.get_env(:pleroma, Pleroma.Upload)
- bucket = Keyword.fetch!(settings, :bucket)
- public_endpoint = Keyword.fetch!(settings, :public_endpoint)
-
- {:ok, file_data} = File.read(path)
-
- File.rm!(path)
-
- s3_name = "#{uuid}/#{name}"
-
- {:ok, result} =
- ExAws.S3.put_object(bucket, s3_name, file_data, [
- {:acl, :public_read},
- {:content_type, content_type}
- ])
- |> ExAws.request()
-
- "#{public_endpoint}/#{bucket}/#{s3_name}"
- end
end
diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex
@@ -0,0 +1,47 @@
+defmodule Pleroma.Uploaders.Local do
+ @behaviour Pleroma.Uploaders.Uploader
+
+ alias Pleroma.Web
+
+ def put_file(name, uuid, tmpfile, _content_type, should_dedupe) do
+ upload_folder = get_upload_path(uuid, should_dedupe)
+ url_path = get_url(name, uuid, should_dedupe)
+
+ File.mkdir_p!(upload_folder)
+
+ result_file = Path.join(upload_folder, name)
+
+ if File.exists?(result_file) do
+ File.rm!(tmpfile)
+ else
+ File.cp!(tmpfile, result_file)
+ end
+
+ {:ok, url_path}
+ end
+
+ def upload_path do
+ settings = Application.get_env(:pleroma, Pleroma.Uploaders.Local)
+ Keyword.fetch!(settings, :uploads)
+ end
+
+ defp get_upload_path(uuid, should_dedupe) do
+ if should_dedupe do
+ upload_path()
+ else
+ Path.join(upload_path(), uuid)
+ end
+ end
+
+ defp get_url(name, uuid, should_dedupe) do
+ if should_dedupe do
+ url_for(:cow_uri.urlencode(name))
+ else
+ url_for(Path.join(uuid, :cow_uri.urlencode(name)))
+ end
+ end
+
+ defp url_for(file) do
+ "#{Web.base_url()}/media/#{file}"
+ end
+end
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
@@ -0,0 +1,24 @@
+defmodule Pleroma.Uploaders.S3 do
+ @behaviour Pleroma.Uploaders.Uploader
+
+ def put_file(name, uuid, path, content_type, _should_dedupe) do
+ settings = Application.get_env(:pleroma, Pleroma.Uploaders.S3)
+ bucket = Keyword.fetch!(settings, :bucket)
+ public_endpoint = Keyword.fetch!(settings, :public_endpoint)
+
+ {:ok, file_data} = File.read(path)
+
+ File.rm!(path)
+
+ s3_name = "#{uuid}/#{name}"
+
+ {:ok, _} =
+ ExAws.S3.put_object(bucket, s3_name, file_data, [
+ {:acl, :public_read},
+ {:content_type, content_type}
+ ])
+ |> ExAws.request()
+
+ {:ok, "#{public_endpoint}/#{bucket}/#{s3_name}"}
+ end
+end
diff --git a/lib/pleroma/uploaders/swift/keystone.ex b/lib/pleroma/uploaders/swift/keystone.ex
@@ -0,0 +1,48 @@
+defmodule Pleroma.Uploaders.Swift.Keystone do
+ use HTTPoison.Base
+
+ @settings Application.get_env(:pleroma, Pleroma.Uploaders.Swift)
+
+ def process_url(url) do
+ Enum.join(
+ [Keyword.fetch!(@settings, :auth_url), url],
+ "/"
+ )
+ end
+
+ def process_response_body(body) do
+ body
+ |> Poison.decode!()
+ end
+
+ def get_token() do
+ username = Keyword.fetch!(@settings, :username)
+ password = Keyword.fetch!(@settings, :password)
+ tenant_id = Keyword.fetch!(@settings, :tenant_id)
+
+ case post(
+ "/tokens",
+ make_auth_body(username, password, tenant_id),
+ ["Content-Type": "application/json"],
+ hackney: [:insecure]
+ ) do
+ {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
+ body["access"]["token"]["id"]
+
+ {:ok, %HTTPoison.Response{status_code: _}} ->
+ ""
+ end
+ end
+
+ def make_auth_body(username, password, tenant) do
+ Poison.encode!(%{
+ :auth => %{
+ :passwordCredentials => %{
+ :username => username,
+ :password => password
+ },
+ :tenantId => tenant
+ }
+ })
+ end
+end
diff --git a/lib/pleroma/uploaders/swift/swift.ex b/lib/pleroma/uploaders/swift/swift.ex
@@ -0,0 +1,28 @@
+defmodule Pleroma.Uploaders.Swift.Client do
+ use HTTPoison.Base
+
+ @settings Application.get_env(:pleroma, Pleroma.Uploaders.Swift)
+
+ def process_url(url) do
+ Enum.join(
+ [Keyword.fetch!(@settings, :storage_url), url],
+ "/"
+ )
+ end
+
+ def upload_file(filename, body, content_type) do
+ object_url = Keyword.fetch!(@settings, :object_url)
+ token = Pleroma.Uploaders.Swift.Keystone.get_token()
+
+ case put("#{filename}", body, "X-Auth-Token": token, "Content-Type": content_type) do
+ {:ok, %HTTPoison.Response{status_code: 201}} ->
+ {:ok, "#{object_url}/#{filename}"}
+
+ {:ok, %HTTPoison.Response{status_code: 401}} ->
+ {:error, "Unauthorized, Bad Token"}
+
+ {:error, _} ->
+ {:error, "Swift Upload Error"}
+ end
+ end
+end
diff --git a/lib/pleroma/uploaders/swift/uploader.ex b/lib/pleroma/uploaders/swift/uploader.ex
@@ -0,0 +1,10 @@
+defmodule Pleroma.Uploaders.Swift do
+ @behaviour Pleroma.Uploaders.Uploader
+
+ def put_file(name, uuid, tmp_path, content_type, _should_dedupe) do
+ {:ok, file_data} = File.read(tmp_path)
+ remote_name = "#{uuid}/#{name}"
+
+ Pleroma.Uploaders.Swift.Client.upload_file(remote_name, file_data, content_type)
+ end
+end
diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex
@@ -0,0 +1,20 @@
+defmodule Pleroma.Uploaders.Uploader do
+ @moduledoc """
+ Defines the contract to put an uploaded file to any backend.
+ """
+
+ @doc """
+ Put a file to the backend.
+
+ Returns `{:ok, String.t } | {:error, String.t} containing the path of the
+ uploaded file, or error information if the file failed to be saved to the
+ respective backend.
+ """
+ @callback put_file(
+ name :: String.t(),
+ uuid :: String.t(),
+ file :: File.t(),
+ content_type :: String.t(),
+ should_dedupe :: Boolean.t()
+ ) :: {:ok, String.t()} | {:error, String.t()}
+end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.Endpoint do
#
# You should set gzip to true if you are running phoenix.digest
# when deploying your static files in production.
- plug(Plug.Static, at: "/media", from: Pleroma.Upload.upload_path(), gzip: false)
+ plug(Plug.Static, at: "/media", from: Pleroma.Uploaders.Local.upload_path(), gzip: false)
plug(
Plug.Static,
