commit: 49b165ddc61091fc192e6f8a252c9a02a5fcef2d
parent: 5ffaa2bf69679f118af9736d62535cc4eb870709
Author: kaniini <nenolod@gmail.com>
Date: Tue, 28 Aug 2018 00:29:49 +0000
Merge branch 'feature/s3' into 'develop'
S3 support
Closes #65
See merge request pleroma/pleroma!303
Diffstat:
5 files changed, 70 insertions(+), 2 deletions(-)
diff --git a/config/config.exs b/config/config.exs
@@ -12,7 +12,9 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
config :pleroma, Pleroma.Upload,
uploads: "uploads",
- strip_exif: false
+ strip_exif: false,
+ use_s3: false,
+ s3_bucket: nil
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex
@@ -24,3 +24,23 @@ config :pleroma, Pleroma.Repo,
database: "pleroma_dev",
hostname: "localhost",
pool_size: 10
+
+# Configure S3 support if desired.
+# The public S3 endpoint is different depending on region and provider,
+# consult your S3 provider's documentation for details on what to use.
+#
+# config :pleroma, Pleroma.Upload,
+# use_s3: true,
+# bucket: "some-bucket",
+# public_endpoint: "https://s3.amazonaws.com"
+#
+# Configure S3 credentials:
+# config :ex_aws, :s3,
+# access_key_id: "xxxxxxxxxxxxx",
+# secret_access_key: "yyyyyyyyyyyy",
+# region: "us-east-1",
+# scheme: "https://"
+#
+# For using third-party S3 clones like wasabi, also do:
+# config :ex_aws, :s3,
+# host: "s3.wasabisys.com"
diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
@@ -3,12 +3,17 @@ defmodule Pleroma.Upload do
alias Pleroma.Web
def store(%Plug.Upload{} = file, should_dedupe) do
+ settings = Application.get_env(:pleroma, Pleroma.Upload)
+ use_s3 = Keyword.fetch!(settings, :use_s3)
+
content_type = get_content_type(file.path)
uuid = get_uuid(file, should_dedupe)
name = get_name(file, uuid, content_type, should_dedupe)
upload_folder = get_upload_path(uuid, should_dedupe)
url_path = get_url(name, uuid, should_dedupe)
+ strip_exif_data(content_type, file.path)
+
File.mkdir_p!(upload_folder)
result_file = Path.join(upload_folder, name)
@@ -18,7 +23,12 @@ defmodule Pleroma.Upload do
File.cp!(file.path, result_file)
end
- strip_exif_data(content_type, result_file)
+ url_path =
+ if use_s3 do
+ put_s3_file(name, uuid, result_file, content_type)
+ else
+ url_path
+ end
%{
"type" => "Document",
@@ -33,7 +43,11 @@ defmodule Pleroma.Upload do
}
end
+ # XXX: does this code actually work? i am skeptical. --kaniini
def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
+ settings = Application.get_env(:pleroma, Pleroma.Upload)
+ use_s3 = Keyword.fetch!(settings, :use_s3)
+
parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
data = Base.decode64!(parsed["data"], ignore: :whitespace)
uuid = UUID.generate()
@@ -71,6 +85,13 @@ defmodule Pleroma.Upload do
strip_exif_data(content_type, result_file)
+ url_path =
+ if use_s3 do
+ put_s3_file(name, uuid, result_file, content_type)
+ else
+ url_path
+ end
+
%{
"type" => "Image",
"url" => [
@@ -203,4 +224,25 @@ defmodule Pleroma.Upload do
_e -> "application/octet-stream"
end
end
+
+ defp put_s3_file(name, uuid, path, content_type) do
+ settings = Application.get_env(:pleroma, Pleroma.Upload)
+ bucket = Keyword.fetch!(settings, :bucket)
+ public_endpoint = Keyword.fetch!(settings, :public_endpoint)
+
+ {:ok, file_data} = File.read(path)
+
+ File.rm!(path)
+
+ s3_name = "#{uuid}/#{name}"
+
+ {:ok, result} =
+ ExAws.S3.put_object(bucket, s3_name, file_data, [
+ {:acl, :public_read},
+ {:content_type, content_type}
+ ])
+ |> ExAws.request()
+
+ "#{public_endpoint}/#{bucket}/#{s3_name}"
+ end
end
diff --git a/mix.exs b/mix.exs
@@ -46,6 +46,8 @@ defmodule Pleroma.Mixfile do
{:httpoison, "~> 1.2.0"},
{:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"},
+ {:ex_aws, "~> 2.0"},
+ {:ex_aws_s3, "~> 2.0"},
{:ex_machina, "~> 2.2", only: :test},
{:credo, "~> 0.9.3", only: [:dev, :test]},
{:mock, "~> 0.3.1", only: :test}
diff --git a/mix.lock b/mix.lock
@@ -12,6 +12,8 @@
"decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], [], "hexpm"},
"ecto": {:hex, :ecto, "2.2.10", "e7366dc82f48f8dd78fcbf3ab50985ceeb11cb3dc93435147c6e13f2cda0992e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
"eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
+ "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
+ "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},
"ex_machina": {:hex, :ex_machina, "2.2.0", "fec496331e04fc2db2a1a24fe317c12c0c4a50d2beb8ebb3531ed1f0d84be0ed", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"},
"hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
