pleroma

commit: 8bec926bebe855e0968f5b71368876cbf2439333
parent 17d885fed87ede236488e80552b9ee9557001e19
Author: Alex Gleason <alex@alexgleason.me>
Date:   Tue,  8 Nov 2022 17:06:16 -0600

LanguageDetector: strip non-language text to (hopefully) improve accuracy

Diffstat:
M
lib/pleroma/language/language_detector.ex
10
+++++++++-

1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/language/language_detector.ex b/lib/pleroma/language/language_detector.ex
@@ -15,10 +15,18 @@ defmodule Pleroma.Language.LanguageDetector do
     end
   end
 
+  # Strip tags from text, etc.
+  defp prepare_text(text) do
+    text
+    |> Floki.parse_fragment!()
+    |> Floki.filter_out(".h-card, .mention, .hashtag, .u-url, .quote-inline, .recipients-inline, code, pre")
+    |> Floki.text()
+  end
+
   def detect(text) do
     provider = get_provider()
 
-    {:ok, text} = text |> FastSanitize.strip_tags()
+    text = prepare_text(text)
     word_count = text |> String.split(~r/\s+/) |> Enum.count()
 
     if word_count < @words_threshold or !provider or !provider.configured? do