logo

WebKitGTK

Collection of patches on top of WebKitGTK

GStreamerRegistryScanner.cpp (18723B)


  1. /*
  2. * Copyright (C) 2019 Igalia S.L
  3. *
  4. * This library is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU Library General Public
  6. * License as published by the Free Software Foundation; either
  7. * version 2 of the License, or (at your option) any later version.
  8. *
  9. * This library is distributed in the hope that it will be useful,
  10. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  12. * Library General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU Library General Public License
  15. * aint with this library; see the file COPYING.LIB. If not, write to
  16. * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  17. * Boston, MA 02110-1301, USA.
  18. */
  19. #include "config.h"
  20. #include "GStreamerRegistryScanner.h"
  21. #if USE(GSTREAMER)
  22. #include "ContentType.h"
  23. #include "GStreamerCommon.h"
  24. #include <fnmatch.h>
  25. #include <gst/pbutils/codec-utils.h>
  26. #include <wtf/PrintStream.h>
  27. namespace WebCore {
  28. GST_DEBUG_CATEGORY_STATIC(webkit_media_gst_registry_scanner_debug);
  29. #define GST_CAT_DEFAULT webkit_media_gst_registry_scanner_debug
  30. GStreamerRegistryScanner& GStreamerRegistryScanner::singleton()
  31. {
  32. static NeverDestroyed<GStreamerRegistryScanner> sharedInstance;
  33. return sharedInstance;
  34. }
  35. GStreamerRegistryScanner::GStreamerRegistryScanner(bool isMediaSource)
  36. : m_isMediaSource(isMediaSource)
  37. {
  38. GST_DEBUG_CATEGORY_INIT(webkit_media_gst_registry_scanner_debug, "webkitregistryscanner", 0, "WebKit GStreamer registry scanner");
  39. m_audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
  40. m_audioParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_NONE);
  41. m_videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
  42. m_videoParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
  43. m_demuxerFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DEMUXER, GST_RANK_MARGINAL);
  44. initialize();
  45. #ifndef GST_DISABLE_GST_DEBUG
  46. GST_DEBUG("%s registry scanner initialized", m_isMediaSource ? "MSE" : "Regular playback");
  47. for (auto& mimeType : m_mimeTypeSet)
  48. GST_DEBUG("Mime-type registered: %s", mimeType.utf8().data());
  49. for (auto& item : m_codecMap)
  50. GST_DEBUG("%s codec pattern registered: %s", item.value ? "Hardware" : "Software", item.key.string().utf8().data());
  51. #endif
  52. }
  53. GStreamerRegistryScanner::~GStreamerRegistryScanner()
  54. {
  55. gst_plugin_feature_list_free(m_audioDecoderFactories);
  56. gst_plugin_feature_list_free(m_audioParserFactories);
  57. gst_plugin_feature_list_free(m_videoDecoderFactories);
  58. gst_plugin_feature_list_free(m_videoParserFactories);
  59. gst_plugin_feature_list_free(m_demuxerFactories);
  60. }
  61. GStreamerRegistryScanner::RegistryLookupResult GStreamerRegistryScanner::hasElementForMediaType(GList* elementFactories, const char* capsString, bool shouldCheckHardwareClassifier) const
  62. {
  63. GRefPtr<GstCaps> caps = adoptGRef(gst_caps_from_string(capsString));
  64. GList* candidates = gst_element_factory_list_filter(elementFactories, caps.get(), GST_PAD_SINK, false);
  65. bool isSupported = candidates;
  66. bool isUsingHardware = false;
  67. if (shouldCheckHardwareClassifier) {
  68. for (GList* factories = candidates; factories; factories = g_list_next(factories)) {
  69. auto* factory = reinterpret_cast<GstElementFactory*>(factories->data);
  70. String metadata = gst_element_factory_get_metadata(factory, GST_ELEMENT_METADATA_KLASS);
  71. auto components = metadata.split('/');
  72. if (components.contains("Hardware")) {
  73. isUsingHardware = true;
  74. break;
  75. }
  76. }
  77. }
  78. gst_plugin_feature_list_free(candidates);
  79. #ifndef GST_DISABLE_GST_DEBUG
  80. const char* elementType = "";
  81. if (elementFactories == m_audioParserFactories)
  82. elementType = "Audio parser";
  83. else if (elementFactories == m_audioDecoderFactories)
  84. elementType = "Audio decoder";
  85. else if (elementFactories == m_videoParserFactories)
  86. elementType = "Video parser";
  87. else if (elementFactories == m_videoDecoderFactories)
  88. elementType = "Video decoder";
  89. else if (elementFactories == m_demuxerFactories)
  90. elementType = "Demuxer";
  91. else
  92. ASSERT_NOT_REACHED();
  93. GST_LOG("%s lookup result for caps %" GST_PTR_FORMAT " : isSupported=%s, isUsingHardware=%s", elementType, caps.get(), boolForPrinting(isSupported), boolForPrinting(isUsingHardware));
  94. #endif
  95. return GStreamerRegistryScanner::RegistryLookupResult { isSupported, isUsingHardware };
  96. }
  97. void GStreamerRegistryScanner::fillMimeTypeSetFromCapsMapping(Vector<GstCapsWebKitMapping>& mapping)
  98. {
  99. for (auto& current : mapping) {
  100. GList* factories;
  101. switch (current.elementType) {
  102. case Demuxer:
  103. factories = m_demuxerFactories;
  104. break;
  105. case AudioDecoder:
  106. factories = m_audioDecoderFactories;
  107. break;
  108. case VideoDecoder:
  109. factories = m_videoDecoderFactories;
  110. break;
  111. }
  112. if (hasElementForMediaType(factories, current.capsString)) {
  113. if (!current.webkitCodecPatterns.isEmpty()) {
  114. for (const auto& pattern : current.webkitCodecPatterns)
  115. m_codecMap.add(pattern, false);
  116. }
  117. if (!current.webkitMimeTypes.isEmpty()) {
  118. for (const auto& mimeType : current.webkitMimeTypes)
  119. m_mimeTypeSet.add(mimeType);
  120. } else
  121. m_mimeTypeSet.add(AtomString(current.capsString));
  122. }
  123. }
  124. }
  125. void GStreamerRegistryScanner::initialize()
  126. {
  127. if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)4")) {
  128. m_mimeTypeSet.add(AtomString("audio/aac"));
  129. m_mimeTypeSet.add(AtomString("audio/mp4"));
  130. m_mimeTypeSet.add(AtomString("audio/x-m4a"));
  131. m_codecMap.add(AtomString("mpeg"), false);
  132. m_codecMap.add(AtomString("mp4a*"), false);
  133. }
  134. auto opusSupported = hasElementForMediaType(m_audioDecoderFactories, "audio/x-opus");
  135. if (opusSupported && (!m_isMediaSource || hasElementForMediaType(m_audioParserFactories, "audio/x-opus"))) {
  136. m_mimeTypeSet.add(AtomString("audio/opus"));
  137. m_codecMap.add(AtomString("opus"), false);
  138. m_codecMap.add(AtomString("x-opus"), false);
  139. }
  140. auto vorbisSupported = hasElementForMediaType(m_audioDecoderFactories, "audio/x-vorbis");
  141. if (vorbisSupported && (!m_isMediaSource || hasElementForMediaType(m_audioParserFactories, "audio/x-vorbis"))) {
  142. m_codecMap.add(AtomString("vorbis"), false);
  143. m_codecMap.add(AtomString("x-vorbis"), false);
  144. }
  145. if (hasElementForMediaType(m_demuxerFactories, "video/x-matroska")) {
  146. auto vp8DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-vp8", true);
  147. auto vp9DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-vp9", true);
  148. if (vp8DecoderAvailable || vp9DecoderAvailable)
  149. m_mimeTypeSet.add(AtomString("video/webm"));
  150. if (vp8DecoderAvailable) {
  151. m_codecMap.add(AtomString("vp8"), vp8DecoderAvailable.isUsingHardware);
  152. m_codecMap.add(AtomString("x-vp8"), vp8DecoderAvailable.isUsingHardware);
  153. m_codecMap.add(AtomString("vp8.0"), vp8DecoderAvailable.isUsingHardware);
  154. }
  155. if (vp9DecoderAvailable) {
  156. m_codecMap.add(AtomString("vp9"), vp9DecoderAvailable.isUsingHardware);
  157. m_codecMap.add(AtomString("x-vp9"), vp9DecoderAvailable.isUsingHardware);
  158. m_codecMap.add(AtomString("vp9.0"), vp9DecoderAvailable.isUsingHardware);
  159. m_codecMap.add(AtomString("vp09*"), vp9DecoderAvailable.isUsingHardware);
  160. }
  161. if (opusSupported)
  162. m_mimeTypeSet.add(AtomString("audio/webm"));
  163. }
  164. auto h264DecoderAvailable = hasElementForMediaType(m_videoDecoderFactories, "video/x-h264, profile=(string){ constrained-baseline, baseline, high }", true);
  165. if (h264DecoderAvailable && (!m_isMediaSource || hasElementForMediaType(m_videoParserFactories, "video/x-h264"))) {
  166. m_mimeTypeSet.add(AtomString("video/mp4"));
  167. m_mimeTypeSet.add(AtomString("video/x-m4v"));
  168. m_codecMap.add(AtomString("x-h264"), h264DecoderAvailable.isUsingHardware);
  169. m_codecMap.add(AtomString("avc*"), h264DecoderAvailable.isUsingHardware);
  170. m_codecMap.add(AtomString("mp4v*"), h264DecoderAvailable.isUsingHardware);
  171. }
  172. if (m_isMediaSource)
  173. return;
  174. // The mime-types initialized below are not supported by the MSE backend.
  175. Vector<GstCapsWebKitMapping> mapping = {
  176. {AudioDecoder, "audio/midi", {"audio/midi", "audio/riff-midi"}, { }},
  177. {AudioDecoder, "audio/x-ac3", { }, { }},
  178. {AudioDecoder, "audio/x-dts", { }, { }},
  179. {AudioDecoder, "audio/x-eac3", {"audio/x-ac3"}, { }},
  180. {AudioDecoder, "audio/x-flac", {"audio/x-flac", "audio/flac"}, { }},
  181. {AudioDecoder, "audio/x-sbc", { }, { }},
  182. {AudioDecoder, "audio/x-sid", { }, { }},
  183. {AudioDecoder, "audio/x-speex", {"audio/speex", "audio/x-speex"}, { }},
  184. {AudioDecoder, "audio/x-wavpack", {"audio/x-wavpack"}, { }},
  185. {AudioDecoder, "audio/x-mod", {"audio/x-mod"}, { }},
  186. {VideoDecoder, "video/mpeg, mpegversion=(int){1,2}, systemstream=(boolean)false", {"video/mpeg"}, {"mpeg"}},
  187. {VideoDecoder, "video/mpegts", { }, { }},
  188. {VideoDecoder, "video/x-dirac", { }, { }},
  189. {VideoDecoder, "video/x-flash-video", {"video/flv", "video/x-flv"}, { }},
  190. {VideoDecoder, "video/x-h263", { }, { }},
  191. {VideoDecoder, "video/x-msvideocodec", {"video/x-msvideo"}, { }},
  192. {Demuxer, "application/vnd.rn-realmedia", { }, { }},
  193. {Demuxer, "application/x-3gp", { }, { }},
  194. {Demuxer, "application/x-dash", {"application/dash+xml"}, { }},
  195. {Demuxer, "application/x-hls", {"application/vnd.apple.mpegurl", "application/x-mpegurl"}, { }},
  196. {Demuxer, "application/x-pn-realaudio", { }, { }},
  197. {Demuxer, "audio/x-aiff", { }, { }},
  198. {Demuxer, "audio/x-wav", {"audio/x-wav", "audio/wav", "audio/vnd.wave"}, {"1"}},
  199. {Demuxer, "video/quicktime", { }, { }},
  200. {Demuxer, "video/quicktime, variant=(string)3gpp", {"video/3gpp"}, { }},
  201. {Demuxer, "video/x-ms-asf", { }, { }},
  202. };
  203. fillMimeTypeSetFromCapsMapping(mapping);
  204. if (hasElementForMediaType(m_demuxerFactories, "application/ogg")) {
  205. m_mimeTypeSet.add(AtomString("application/ogg"));
  206. if (vorbisSupported) {
  207. m_mimeTypeSet.add(AtomString("audio/ogg"));
  208. m_mimeTypeSet.add(AtomString("audio/x-vorbis+ogg"));
  209. }
  210. if (hasElementForMediaType(m_audioDecoderFactories, "audio/x-speex")) {
  211. m_mimeTypeSet.add(AtomString("audio/ogg"));
  212. m_codecMap.add(AtomString("speex"), false);
  213. }
  214. if (hasElementForMediaType(m_videoDecoderFactories, "video/x-theora")) {
  215. m_mimeTypeSet.add(AtomString("video/ogg"));
  216. m_codecMap.add(AtomString("theora"), false);
  217. }
  218. }
  219. bool audioMpegSupported = false;
  220. if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)1, layer=(int)[1, 3]")) {
  221. audioMpegSupported = true;
  222. m_mimeTypeSet.add(AtomString("audio/mp1"));
  223. m_mimeTypeSet.add(AtomString("audio/mp3"));
  224. m_mimeTypeSet.add(AtomString("audio/x-mp3"));
  225. m_codecMap.add(AtomString("audio/mp3"), false);
  226. }
  227. if (hasElementForMediaType(m_audioDecoderFactories, "audio/mpeg, mpegversion=(int)2")) {
  228. audioMpegSupported = true;
  229. m_mimeTypeSet.add(AtomString("audio/mp2"));
  230. }
  231. audioMpegSupported |= isContainerTypeSupported("audio/mp4");
  232. if (audioMpegSupported) {
  233. m_mimeTypeSet.add(AtomString("audio/mpeg"));
  234. m_mimeTypeSet.add(AtomString("audio/x-mpeg"));
  235. }
  236. bool matroskaSupported = hasElementForMediaType(m_demuxerFactories, "video/x-matroska");
  237. if (matroskaSupported) {
  238. m_mimeTypeSet.add(AtomString("video/x-matroska"));
  239. if (hasElementForMediaType(m_videoDecoderFactories, "video/x-vp10"))
  240. m_mimeTypeSet.add(AtomString("video/webm"));
  241. }
  242. if ((matroskaSupported || isContainerTypeSupported("video/mp4")) && hasElementForMediaType(m_videoDecoderFactories, "video/x-av1"))
  243. m_codecMap.add(AtomString("av01*"), false);
  244. }
  245. bool GStreamerRegistryScanner::isCodecSupported(String codec, bool shouldCheckForHardwareUse) const
  246. {
  247. // If the codec is named like a mimetype (eg: video/avc) remove the "video/" part.
  248. size_t slashIndex = codec.find('/');
  249. if (slashIndex != WTF::notFound)
  250. codec = codec.substring(slashIndex + 1);
  251. bool supported = false;
  252. if (codec.startsWith("avc1"))
  253. supported = isAVC1CodecSupported(codec, shouldCheckForHardwareUse);
  254. else {
  255. for (const auto& item : m_codecMap) {
  256. if (!fnmatch(item.key.string().utf8().data(), codec.utf8().data(), 0)) {
  257. supported = shouldCheckForHardwareUse ? item.value : true;
  258. if (supported)
  259. break;
  260. }
  261. }
  262. }
  263. GST_LOG("Checked %s codec \"%s\" supported %s", shouldCheckForHardwareUse ? "hardware" : "software", codec.utf8().data(), boolForPrinting(supported));
  264. return supported;
  265. }
  266. bool GStreamerRegistryScanner::areAllCodecsSupported(const Vector<String>& codecs, bool shouldCheckForHardwareUse) const
  267. {
  268. for (String codec : codecs) {
  269. if (!isCodecSupported(codec, shouldCheckForHardwareUse))
  270. return false;
  271. }
  272. return true;
  273. }
  274. bool GStreamerRegistryScanner::isAVC1CodecSupported(const String& codec, bool shouldCheckForHardwareUse) const
  275. {
  276. auto components = codec.split('.');
  277. long int spsAsInteger = strtol(components[1].utf8().data(), nullptr, 16);
  278. uint8_t sps[3];
  279. sps[0] = spsAsInteger >> 16;
  280. sps[1] = spsAsInteger >> 8;
  281. sps[2] = spsAsInteger;
  282. const char* profile = gst_codec_utils_h264_get_profile(sps, 3);
  283. const char* level = gst_codec_utils_h264_get_level(sps, 3);
  284. // To avoid going through a class hierarchy for such a simple
  285. // string conversion, we use a little trick here: See
  286. // https://bugs.webkit.org/show_bug.cgi?id=201870.
  287. char levelAsStringFallback[2] = { '\0', '\0' };
  288. if (!level && sps[2] > 0 && sps[2] <= 5) {
  289. levelAsStringFallback[0] = static_cast<char>('0' + sps[2]);
  290. level = levelAsStringFallback;
  291. }
  292. if (!profile || !level) {
  293. GST_ERROR("H.264 profile / level was not recognised in codec %s", codec.utf8().data());
  294. return false;
  295. }
  296. GST_DEBUG("Codec %s translates to H.264 profile %s and level %s", codec.utf8().data(), profile, level);
  297. auto checkH264Caps = [&](const char* capsString) {
  298. bool supported = false;
  299. auto lookupResult = hasElementForMediaType(m_videoDecoderFactories, capsString, true);
  300. supported = lookupResult;
  301. if (shouldCheckForHardwareUse)
  302. supported = lookupResult.isUsingHardware;
  303. GST_DEBUG("%s decoding supported for codec %s: %s", shouldCheckForHardwareUse ? "Hardware" : "Software", codec.utf8().data(), boolForPrinting(supported));
  304. return supported;
  305. };
  306. if (const char* maxVideoResolution = g_getenv("WEBKIT_GST_MAX_AVC1_RESOLUTION")) {
  307. uint8_t levelAsInteger = gst_codec_utils_h264_get_level_idc(level);
  308. GST_DEBUG("Maximum video resolution requested: %s, supplied codec level IDC: %u", maxVideoResolution, levelAsInteger);
  309. uint8_t maxLevel = 0;
  310. const char* maxLevelString = "";
  311. if (!g_strcmp0(maxVideoResolution, "1080P")) {
  312. maxLevel = 40;
  313. maxLevelString = "4";
  314. } else if (!g_strcmp0(maxVideoResolution, "720P")) {
  315. maxLevel = 31;
  316. maxLevelString = "3.1";
  317. } else if (!g_strcmp0(maxVideoResolution, "480P")) {
  318. maxLevel = 30;
  319. maxLevelString = "3";
  320. } else {
  321. g_warning("Invalid value for WEBKIT_GST_MAX_AVC1_RESOLUTION. Currently supported, 1080P, 720P and 480P.");
  322. return false;
  323. }
  324. if (levelAsInteger > maxLevel)
  325. return false;
  326. return checkH264Caps(makeString("video/x-h264, level=(string)", maxLevelString).utf8().data());
  327. }
  328. if (webkitGstCheckVersion(1, 17, 0)) {
  329. GST_DEBUG("Checking video decoders for constrained caps");
  330. return checkH264Caps(makeString("video/x-h264, level=(string)", level, ", profile=(string)", profile).utf8().data());
  331. }
  332. GST_DEBUG("Falling back to unconstrained caps");
  333. return checkH264Caps("video/x-h264");
  334. }
  335. GStreamerRegistryScanner::RegistryLookupResult GStreamerRegistryScanner::isDecodingSupported(MediaConfiguration& configuration) const
  336. {
  337. bool isSupported = false;
  338. bool isUsingHardware = false;
  339. if (configuration.video) {
  340. auto& videoConfiguration = configuration.video.value();
  341. GST_DEBUG("Checking support for video configuration: \"%s\" size: %ux%u bitrate: %" G_GUINT64_FORMAT " framerate: %f",
  342. videoConfiguration.contentType.utf8().data(),
  343. videoConfiguration.width, videoConfiguration.height,
  344. videoConfiguration.bitrate, videoConfiguration.framerate);
  345. auto contentType = ContentType(videoConfiguration.contentType);
  346. isSupported = isContainerTypeSupported(contentType.containerType());
  347. auto codecs = contentType.codecs();
  348. if (!codecs.isEmpty())
  349. isUsingHardware = areAllCodecsSupported(codecs, true);
  350. }
  351. if (configuration.audio) {
  352. auto& audioConfiguration = configuration.audio.value();
  353. GST_DEBUG("Checking support for audio configuration: \"%s\" %s channels, bitrate: %" G_GUINT64_FORMAT " samplerate: %u",
  354. audioConfiguration.contentType.utf8().data(), audioConfiguration.channels.utf8().data(),
  355. audioConfiguration.bitrate, audioConfiguration.samplerate);
  356. auto contentType = ContentType(audioConfiguration.contentType);
  357. isSupported = isContainerTypeSupported(contentType.containerType());
  358. }
  359. return GStreamerRegistryScanner::RegistryLookupResult { isSupported, isUsingHardware };
  360. }
  361. }
  362. #endif