commit: fd7f0732fe26554c51218c4f67955e8050590d2c parent: eb5ac234342db46c881d8e69644d3292b5eabb54 Author: Nolan Lawson <nolan@nolanlawson.com> Date: Thu, 5 Oct 2017 18:42:34 -0700 Compress and combine emoji data (#5229)Diffstat:
25 files changed, 611 insertions(+), 450 deletions(-)
diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js @@ -1,6 +1,6 @@ import api from '../api'; import { throttle } from 'lodash'; -import { search as emojiSearch } from '../emoji_index_light'; +import { search as emojiSearch } from '../features/emoji/emoji_mart_search_light'; import { updateTimeline, diff --git a/app/javascript/mastodon/components/autosuggest_emoji.js b/app/javascript/mastodon/components/autosuggest_emoji.js @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { unicodeMapping } from '../emojione_light'; +import unicodeMapping from '../features/emoji/emoji_unicode_mapping_light'; const assetHost = process.env.CDN_HOST || ''; @@ -23,7 +23,7 @@ export default class AutosuggestEmoji extends React.PureComponent { return null; } - url = `${assetHost}/emoji/${mapping[0]}.svg`; + url = `${assetHost}/emoji/${mapping.filename}.svg`; } return ( diff --git a/app/javascript/mastodon/emoji.js b/app/javascript/mastodon/emoji.js @@ -1,71 +0,0 @@ -import { unicodeMapping } from './emojione_light'; -import Trie from 'substring-trie'; - -const trie = new Trie(Object.keys(unicodeMapping)); - -const assetHost = process.env.CDN_HOST || ''; - -const emojify = (str, customEmojis = {}) => { - let rtn = ''; - for (;;) { - let match, i = 0, tag; - while (i < str.length && (tag = '<&:'.indexOf(str[i])) === -1 && !(match = trie.search(str.slice(i)))) { - i += str.codePointAt(i) < 65536 ? 1 : 2; - } - let rend, replacement = ''; - if (i === str.length) { - break; - } else if (str[i] === ':') { - if (!(() => { - rend = str.indexOf(':', i + 1) + 1; - if (!rend) return false; // no pair of ':' - const lt = str.indexOf('<', i + 1); - if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':' - const shortname = str.slice(i, rend); - // now got a replacee as ':shortname:' - // if you want additional emoji handler, add statements below which set replacement and return true. - if (shortname in customEmojis) { - replacement = `<img draggable="false" class="emojione" alt="${shortname}" title="${shortname}" src="${customEmojis[shortname]}" />`; - return true; - } - return false; - })()) rend = ++i; - } else if (tag >= 0) { // <, & - rend = str.indexOf('>;'[tag], i + 1) + 1; - if (!rend) break; - i = rend; - } else { // matched to unicode emoji - const [filename, shortCode] = unicodeMapping[match]; - replacement = `<img draggable="false" class="emojione" alt="${match}" title=":${shortCode}:" src="${assetHost}/emoji/${filename}.svg" />`; - rend = i + match.length; - } - rtn += str.slice(0, i) + replacement; - str = str.slice(rend); - } - return rtn + str; -}; - -export default emojify; - -export const buildCustomEmojis = customEmojis => { - const emojis = []; - - customEmojis.forEach(emoji => { - const shortcode = emoji.get('shortcode'); - const url = emoji.get('static_url'); - const name = shortcode.replace(':', ''); - - emojis.push({ - id: name, - name, - short_names: [name], - text: '', - emoticons: [], - keywords: [name], - imageUrl: url, - custom: true, - }); - }); - - return emojis; -}; diff --git a/app/javascript/mastodon/emoji_data_compressed.js b/app/javascript/mastodon/emoji_data_compressed.js @@ -1,22 +0,0 @@ -// @preval -const data = require('emoji-mart/dist/data').default; -const pick = require('lodash/pick'); -const values = require('lodash/values'); - -const condensedEmojis = Object.keys(data.emojis).map(key => { - if (!data.emojis[key].short_names[0] === key) { - throw new Error('The condenser expects the first short_code to be the ' + - 'key. It may need to be rewritten if the emoji change such that this ' + - 'is no longer the case.'); - } - return values(pick(data.emojis[key], ['short_names', 'unified', 'search'])); -}); - -// JSON.parse/stringify is to emulate what @preval is doing and avoid any -// inconsistent behavior in dev mode -module.exports = JSON.parse(JSON.stringify({ - emojis: condensedEmojis, - skins: data.skins, - categories: data.categories, - short_names: data.short_names, -})); diff --git a/app/javascript/mastodon/emoji_data_light.js b/app/javascript/mastodon/emoji_data_light.js @@ -1,16 +0,0 @@ -const data = require('./emoji_data_compressed'); - -// decompress -const emojis = {}; -data.emojis.forEach(compressedEmoji => { - const [ short_names, unified, search ] = compressedEmoji; - emojis[short_names[0]] = { - short_names, - unified, - search, - }; -}); - -data.emojis = emojis; - -module.exports = data; diff --git a/app/javascript/mastodon/emoji_index_light.js b/app/javascript/mastodon/emoji_index_light.js @@ -1,154 +0,0 @@ -// This code is largely borrowed from: -// https://github.com/missive/emoji-mart/blob/bbd4fbe/src/utils/emoji-index.js - -import data from './emoji_data_light'; -import { getData, getSanitizedData, intersect } from './emoji_utils'; - -let index = {}; -let emojisList = {}; -let emoticonsList = {}; -let previousInclude = []; -let previousExclude = []; - -for (let emoji in data.emojis) { - let emojiData = data.emojis[emoji], - { short_names, emoticons } = emojiData, - id = short_names[0]; - - for (let emoticon of (emoticons || [])) { - if (!emoticonsList[emoticon]) { - emoticonsList[emoticon] = id; - } - } - - emojisList[id] = getSanitizedData(id); -} - -function search(value, { emojisToShowFilter, maxResults, include, exclude, custom = [] } = {}) { - maxResults = maxResults || 75; - include = include || []; - exclude = exclude || []; - - if (custom.length) { - for (const emoji of custom) { - data.emojis[emoji.id] = getData(emoji); - emojisList[emoji.id] = getSanitizedData(emoji); - } - - data.categories.push({ - name: 'Custom', - emojis: custom.map(emoji => emoji.id), - }); - } - - let results = null; - let pool = data.emojis; - - if (value.length) { - if (value === '-' || value === '-1') { - return [emojisList['-1']]; - } - - let values = value.toLowerCase().split(/[\s|,|\-|_]+/); - - if (values.length > 2) { - values = [values[0], values[1]]; - } - - if (include.length || exclude.length) { - pool = {}; - - if (previousInclude !== include.sort().join(',') || previousExclude !== exclude.sort().join(',')) { - previousInclude = include.sort().join(','); - previousExclude = exclude.sort().join(','); - index = {}; - } - - for (let category of data.categories) { - let isIncluded = include && include.length ? include.indexOf(category.name.toLowerCase()) > -1 : true; - let isExcluded = exclude && exclude.length ? exclude.indexOf(category.name.toLowerCase()) > -1 : false; - if (!isIncluded || isExcluded) { - continue; - } - - for (let emojiId of category.emojis) { - pool[emojiId] = data.emojis[emojiId]; - } - } - } else if (previousInclude.length || previousExclude.length) { - index = {}; - } - - let allResults = values.map((value) => { - let aPool = pool; - let aIndex = index; - let length = 0; - - for (let char of value.split('')) { - length++; - - aIndex[char] = aIndex[char] || {}; - aIndex = aIndex[char]; - - if (!aIndex.results) { - let scores = {}; - - aIndex.results = []; - aIndex.pool = {}; - - for (let id in aPool) { - let emoji = aPool[id], - { search } = emoji, - sub = value.substr(0, length), - subIndex = search.indexOf(sub); - - if (subIndex !== -1) { - let score = subIndex + 1; - if (sub === id) { - score = 0; - } - - aIndex.results.push(emojisList[id]); - aIndex.pool[id] = emoji; - - scores[id] = score; - } - } - - aIndex.results.sort((a, b) => { - let aScore = scores[a.id], - bScore = scores[b.id]; - - return aScore - bScore; - }); - } - - aPool = aIndex.pool; - } - - return aIndex.results; - }).filter(a => a); - - if (allResults.length > 1) { - results = intersect(...allResults); - } else if (allResults.length) { - results = allResults[0]; - } else { - results = []; - } - } - - if (results) { - if (emojisToShowFilter) { - results = results.filter((result) => emojisToShowFilter(data.emojis[result.id].unified)); - } - - if (results && results.length > maxResults) { - results = results.slice(0, maxResults); - } - } - - return results; -} - -export { search }; diff --git a/app/javascript/mastodon/emoji_utils.js b/app/javascript/mastodon/emoji_utils.js @@ -1,137 +0,0 @@ -// This code is largely borrowed from: -// https://github.com/missive/emoji-mart/blob/bbd4fbe/src/utils/index.js - -import data from './emoji_data_light'; - -const COLONS_REGEX = /^(?:\:([^\:]+)\:)(?:\:skin-tone-(\d)\:)?$/; - -function buildSearch(thisData) { - const search = []; - - let addToSearch = (strings, split) => { - if (!strings) { - return; - } - - (Array.isArray(strings) ? strings : [strings]).forEach((string) => { - (split ? string.split(/[-|_|\s]+/) : [string]).forEach((s) => { - s = s.toLowerCase(); - - if (search.indexOf(s) === -1) { - search.push(s); - } - }); - }); - }; - - addToSearch(thisData.short_names, true); - addToSearch(thisData.name, true); - addToSearch(thisData.keywords, false); - addToSearch(thisData.emoticons, false); - - return search; -} - -function unifiedToNative(unified) { - let unicodes = unified.split('-'), - codePoints = unicodes.map((u) => `0x${u}`); - - return String.fromCodePoint(...codePoints); -} - -function sanitize(emoji) { - let { name, short_names, skin_tone, skin_variations, emoticons, unified, custom, imageUrl } = emoji, - id = emoji.id || short_names[0], - colons = `:${id}:`; - - if (custom) { - return { - id, - name, - colons, - emoticons, - custom, - imageUrl, - }; - } - - if (skin_tone) { - colons += `:skin-tone-${skin_tone}:`; - } - - return { - id, - name, - colons, - emoticons, - unified: unified.toLowerCase(), - skin: skin_tone || (skin_variations ? 1 : null), - native: unifiedToNative(unified), - }; -} - -function getSanitizedData(emoji) { - return sanitize(getData(emoji)); -} - -function getData(emoji) { - let emojiData = {}; - - if (typeof emoji === 'string') { - let matches = emoji.match(COLONS_REGEX); - - if (matches) { - emoji = matches[1]; - - } - - if (data.short_names.hasOwnProperty(emoji)) { - emoji = data.short_names[emoji]; - } - - if (data.emojis.hasOwnProperty(emoji)) { - emojiData = data.emojis[emoji]; - } - } else if (emoji.custom) { - emojiData = emoji; - - emojiData.search = buildSearch({ - short_names: emoji.short_names, - name: emoji.name, - keywords: emoji.keywords, - emoticons: emoji.emoticons, - }); - - emojiData.search = emojiData.search.join(','); - } else if (emoji.id) { - if (data.short_names.hasOwnProperty(emoji.id)) { - emoji.id = data.short_names[emoji.id]; - } - - if (data.emojis.hasOwnProperty(emoji.id)) { - emojiData = data.emojis[emoji.id]; - } - } - - emojiData.emoticons = emojiData.emoticons || []; - emojiData.variations = emojiData.variations || []; - - if (emojiData.variations && emojiData.variations.length) { - emojiData = JSON.parse(JSON.stringify(emojiData)); - emojiData.unified = emojiData.variations.shift(); - } - - return emojiData; -} - -function intersect(a, b) { - let aSet = new Set(a); - let bSet = new Set(b); - let intersection = new Set( - [...aSet].filter(x => bSet.has(x)) - ); - - return Array.from(intersection); -} - -export { getData, getSanitizedData, intersect }; diff --git a/app/javascript/mastodon/emojione_light.js b/app/javascript/mastodon/emojione_light.js @@ -1,38 +0,0 @@ -// @preval -// http://www.unicode.org/Public/emoji/5.0/emoji-test.txt - -const emojis = require('./emoji_map.json'); -const { emojiIndex } = require('emoji-mart'); -const excluded = ['®', '©', '™']; -const skins = ['🏻', '🏼', '🏽', '🏾', '🏿']; -const shortcodeMap = {}; - -Object.keys(emojiIndex.emojis).forEach(key => { - shortcodeMap[emojiIndex.emojis[key].native] = emojiIndex.emojis[key].id; -}); - -const stripModifiers = unicode => { - skins.forEach(tone => { - unicode = unicode.replace(tone, ''); - }); - - return unicode; -}; - -Object.keys(emojis).forEach(key => { - if (excluded.includes(key)) { - delete emojis[key]; - return; - } - - const normalizedKey = stripModifiers(key); - let shortcode = shortcodeMap[normalizedKey]; - - if (!shortcode) { - shortcode = shortcodeMap[normalizedKey + '\uFE0F']; - } - - emojis[key] = [emojis[key], shortcode]; -}); - -module.exports.unicodeMapping = emojis; diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js @@ -6,7 +6,7 @@ import Overlay from 'react-overlays/lib/Overlay'; import classNames from 'classnames'; import ImmutablePropTypes from 'react-immutable-proptypes'; import detectPassiveEvents from 'detect-passive-events'; -import { buildCustomEmojis } from '../../../emoji'; +import { buildCustomEmojis } from '../../emoji/emoji'; const messages = defineMessages({ emoji: { id: 'emoji_button.label', defaultMessage: 'Insert emoji' }, diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js @@ -0,0 +1,72 @@ +import unicodeMapping from './emoji_unicode_mapping_light'; +import Trie from 'substring-trie'; + +const trie = new Trie(Object.keys(unicodeMapping)); + +const assetHost = process.env.CDN_HOST || ''; + +const emojify = (str, customEmojis = {}) => { + let rtn = ''; + for (;;) { + let match, i = 0, tag; + while (i < str.length && (tag = '<&:'.indexOf(str[i])) === -1 && !(match = trie.search(str.slice(i)))) { + i += str.codePointAt(i) < 65536 ? 1 : 2; + } + let rend, replacement = ''; + if (i === str.length) { + break; + } else if (str[i] === ':') { + if (!(() => { + rend = str.indexOf(':', i + 1) + 1; + if (!rend) return false; // no pair of ':' + const lt = str.indexOf('<', i + 1); + if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':' + const shortname = str.slice(i, rend); + // now got a replacee as ':shortname:' + // if you want additional emoji handler, add statements below which set replacement and return true. + if (shortname in customEmojis) { + replacement = `<img draggable="false" class="emojione" alt="${shortname}" title="${shortname}" src="${customEmojis[shortname]}" />`; + return true; + } + return false; + })()) rend = ++i; + } else if (tag >= 0) { // <, & + rend = str.indexOf('>;'[tag], i + 1) + 1; + if (!rend) break; + i = rend; + } else { // matched to unicode emoji + const { filename, shortCode } = unicodeMapping[match]; + const title = shortCode ? `:${shortCode}:` : ''; + replacement = `<img draggable="false" class="emojione" alt="${match}" title="${title}" src="${assetHost}/emoji/${filename}.svg" />`; + rend = i + match.length; + } + rtn += str.slice(0, i) + replacement; + str = str.slice(rend); + } + return rtn + str; +}; + +export default emojify; + +export const buildCustomEmojis = customEmojis => { + const emojis = []; + + customEmojis.forEach(emoji => { + const shortcode = emoji.get('shortcode'); + const url = emoji.get('static_url'); + const name = shortcode.replace(':', ''); + + emojis.push({ + id: name, + name, + short_names: [name], + text: '', + emoticons: [], + keywords: [name], + imageUrl: url, + custom: true, + }); + }); + + return emojis; +}; diff --git a/app/javascript/mastodon/features/emoji/emoji_compressed.js b/app/javascript/mastodon/features/emoji/emoji_compressed.js @@ -0,0 +1,90 @@ +// @preval +// http://www.unicode.org/Public/emoji/5.0/emoji-test.txt +// This file contains the compressed version of the emoji data from +// both emoji_map.json and from emoji-mart's emojiIndex and data objects. +// It's designed to be emitted in an array format to take up less space +// over the wire. + +const { unicodeToFilename } = require('./unicode_to_filename'); +const { unicodeToUnifiedName } = require('./unicode_to_unified_name'); +const emojiMap = require('./emoji_map.json'); +const { emojiIndex } = require('emoji-mart'); +const emojiMartData = require('emoji-mart/dist/data').default; +const excluded = ['®', '©', '™']; +const skins = ['🏻', '🏼', '🏽', '🏾', '🏿']; +const shortcodeMap = {}; + +const shortCodesToEmojiData = {}; +const emojisWithoutShortCodes = []; + +Object.keys(emojiIndex.emojis).forEach(key => { + shortcodeMap[emojiIndex.emojis[key].native] = emojiIndex.emojis[key].id; +}); + +const stripModifiers = unicode => { + skins.forEach(tone => { + unicode = unicode.replace(tone, ''); + }); + + return unicode; +}; + +Object.keys(emojiMap).forEach(key => { + if (excluded.includes(key)) { + delete emojiMap[key]; + return; + } + + const normalizedKey = stripModifiers(key); + let shortcode = shortcodeMap[normalizedKey]; + + if (!shortcode) { + shortcode = shortcodeMap[normalizedKey + '\uFE0F']; + } + + const filename = emojiMap[key]; + + const filenameData = [key]; + + if (unicodeToFilename(key) !== filename) { + // filename can't be derived using unicodeToFilename + filenameData.push(filename); + } + + if (typeof shortcode === 'undefined') { + emojisWithoutShortCodes.push(filenameData); + } else { + shortCodesToEmojiData[shortcode] = shortCodesToEmojiData[shortcode] || [[]]; + shortCodesToEmojiData[shortcode][0].push(filenameData); + } +}); + +Object.keys(emojiIndex.emojis).forEach(key => { + const { native } = emojiIndex.emojis[key]; + const { short_names, search, unified } = emojiMartData.emojis[key]; + if (short_names[0] !== key) { + throw new Error('The compresser expects the first short_code to be the ' + + 'key. It may need to be rewritten if the emoji change such that this ' + + 'is no longer the case.'); + } + + short_names.splice(0, 1); // first short name can be inferred from the key + + const searchData = [native, short_names, search]; + if (unicodeToUnifiedName(native) !== unified) { + // unified name can't be derived from unicodeToUnifiedName + searchData.push(unified); + } + + shortCodesToEmojiData[key].push(searchData); +}); + +// JSON.parse/stringify is to emulate what @preval is doing and avoid any +// inconsistent behavior in dev mode +module.exports = JSON.parse(JSON.stringify([ + shortCodesToEmojiData, + emojiMartData.skins, + emojiMartData.categories, + emojiMartData.short_names, + emojisWithoutShortCodes, +])); diff --git a/app/javascript/mastodon/emoji_map.json b/app/javascript/mastodon/features/emoji/emoji_map.json diff --git a/app/javascript/mastodon/features/emoji/emoji_mart_data_light.js b/app/javascript/mastodon/features/emoji/emoji_mart_data_light.js @@ -0,0 +1,41 @@ +// The output of this module is designed to mimic emoji-mart's +// "data" object, such that we can use it for a light version of emoji-mart's +// emojiIndex.search functionality. +const { unicodeToUnifiedName } = require('./unicode_to_unified_name'); +const [ shortCodesToEmojiData, skins, categories, short_names ] = require('./emoji_compressed'); + +const emojis = {}; + +// decompress +Object.keys(shortCodesToEmojiData).forEach((shortCode) => { + let [ + filenameData, // eslint-disable-line no-unused-vars + searchData, + ] = shortCodesToEmojiData[shortCode]; + let [ + native, + short_names, + search, + unified, + ] = searchData; + + if (!unified) { + // unified name can be derived from unicodeToUnifiedName + unified = unicodeToUnifiedName(native); + } + + short_names = [shortCode].concat(short_names); + emojis[shortCode] = { + native, + search, + short_names, + unified, + }; +}); + +module.exports = { + emojis, + skins, + categories, + short_names, +}; diff --git a/app/javascript/mastodon/features/emoji/emoji_mart_search_light.js b/app/javascript/mastodon/features/emoji/emoji_mart_search_light.js @@ -0,0 +1,154 @@ +// This code is largely borrowed from: +// https://github.com/missive/emoji-mart/blob/bbd4fbe/src/utils/emoji-index.js + +import data from './emoji_mart_data_light'; +import { getData, getSanitizedData, intersect } from './emoji_utils'; + +let index = {}; +let emojisList = {}; +let emoticonsList = {}; +let previousInclude = []; +let previousExclude = []; + +for (let emoji in data.emojis) { + let emojiData = data.emojis[emoji], + { short_names, emoticons } = emojiData, + id = short_names[0]; + + for (let emoticon of (emoticons || [])) { + if (!emoticonsList[emoticon]) { + emoticonsList[emoticon] = id; + } + } + + emojisList[id] = getSanitizedData(id); +} + +function search(value, { emojisToShowFilter, maxResults, include, exclude, custom = [] } = {}) { + maxResults = maxResults || 75; + include = include || []; + exclude = exclude || []; + + if (custom.length) { + for (const emoji of custom) { + data.emojis[emoji.id] = getData(emoji); + emojisList[emoji.id] = getSanitizedData(emoji); + } + + data.categories.push({ + name: 'Custom', + emojis: custom.map(emoji => emoji.id), + }); + } + + let results = null; + let pool = data.emojis; + + if (value.length) { + if (value === '-' || value === '-1') { + return [emojisList['-1']]; + } + + let values = value.toLowerCase().split(/[\s|,|\-|_]+/); + + if (values.length > 2) { + values = [values[0], values[1]]; + } + + if (include.length || exclude.length) { + pool = {}; + + if (previousInclude !== include.sort().join(',') || previousExclude !== exclude.sort().join(',')) { + previousInclude = include.sort().join(','); + previousExclude = exclude.sort().join(','); + index = {}; + } + + for (let category of data.categories) { + let isIncluded = include && include.length ? include.indexOf(category.name.toLowerCase()) > -1 : true; + let isExcluded = exclude && exclude.length ? exclude.indexOf(category.name.toLowerCase()) > -1 : false; + if (!isIncluded || isExcluded) { + continue; + } + + for (let emojiId of category.emojis) { + pool[emojiId] = data.emojis[emojiId]; + } + } + } else if (previousInclude.length || previousExclude.length) { + index = {}; + } + + let allResults = values.map((value) => { + let aPool = pool; + let aIndex = index; + let length = 0; + + for (let char of value.split('')) { + length++; + + aIndex[char] = aIndex[char] || {}; + aIndex = aIndex[char]; + + if (!aIndex.results) { + let scores = {}; + + aIndex.results = []; + aIndex.pool = {}; + + for (let id in aPool) { + let emoji = aPool[id], + { search } = emoji, + sub = value.substr(0, length), + subIndex = search.indexOf(sub); + + if (subIndex !== -1) { + let score = subIndex + 1; + if (sub === id) { + score = 0; + } + + aIndex.results.push(emojisList[id]); + aIndex.pool[id] = emoji; + + scores[id] = score; + } + } + + aIndex.results.sort((a, b) => { + let aScore = scores[a.id], + bScore = scores[b.id]; + + return aScore - bScore; + }); + } + + aPool = aIndex.pool; + } + + return aIndex.results; + }).filter(a => a); + + if (allResults.length > 1) { + results = intersect(...allResults); + } else if (allResults.length) { + results = allResults[0]; + } else { + results = []; + } + } + + if (results) { + if (emojisToShowFilter) { + results = results.filter((result) => emojisToShowFilter(data.emojis[result.id].unified)); + } + + if (results && results.length > maxResults) { + results = results.slice(0, maxResults); + } + } + + return results; +} + +export { search }; diff --git a/app/javascript/mastodon/features/emoji/emoji_unicode_mapping_light.js b/app/javascript/mastodon/features/emoji/emoji_unicode_mapping_light.js @@ -0,0 +1,35 @@ +// A mapping of unicode strings to an object containing the filename +// (i.e. the svg filename) and a shortCode intended to be shown +// as a "title" attribute in an HTML element (aka tooltip). + +const [ + shortCodesToEmojiData, + skins, // eslint-disable-line no-unused-vars + categories, // eslint-disable-line no-unused-vars + short_names, // eslint-disable-line no-unused-vars + emojisWithoutShortCodes, +] = require('./emoji_compressed'); +const { unicodeToFilename } = require('./unicode_to_filename'); + +// decompress +const unicodeMapping = {}; + +function processEmojiMapData(emojiMapData, shortCode) { + let [ native, filename ] = emojiMapData; + if (!filename) { + // filename name can be derived from unicodeToFilename + filename = unicodeToFilename(native); + } + unicodeMapping[native] = { + shortCode: shortCode, + filename: filename, + }; +} + +Object.keys(shortCodesToEmojiData).forEach((shortCode) => { + let [ filenameData ] = shortCodesToEmojiData[shortCode]; + filenameData.forEach(emojiMapData => processEmojiMapData(emojiMapData, shortCode)); +}); +emojisWithoutShortCodes.forEach(emojiMapData => processEmojiMapData(emojiMapData)); + +module.exports = unicodeMapping; diff --git a/app/javascript/mastodon/features/emoji/emoji_utils.js b/app/javascript/mastodon/features/emoji/emoji_utils.js @@ -0,0 +1,137 @@ +// This code is largely borrowed from: +// https://github.com/missive/emoji-mart/blob/bbd4fbe/src/utils/index.js + +import data from './emoji_mart_data_light'; + +const COLONS_REGEX = /^(?:\:([^\:]+)\:)(?:\:skin-tone-(\d)\:)?$/; + +function buildSearch(thisData) { + const search = []; + + let addToSearch = (strings, split) => { + if (!strings) { + return; + } + + (Array.isArray(strings) ? strings : [strings]).forEach((string) => { + (split ? string.split(/[-|_|\s]+/) : [string]).forEach((s) => { + s = s.toLowerCase(); + + if (search.indexOf(s) === -1) { + search.push(s); + } + }); + }); + }; + + addToSearch(thisData.short_names, true); + addToSearch(thisData.name, true); + addToSearch(thisData.keywords, false); + addToSearch(thisData.emoticons, false); + + return search; +} + +function unifiedToNative(unified) { + let unicodes = unified.split('-'), + codePoints = unicodes.map((u) => `0x${u}`); + + return String.fromCodePoint(...codePoints); +} + +function sanitize(emoji) { + let { name, short_names, skin_tone, skin_variations, emoticons, unified, custom, imageUrl } = emoji, + id = emoji.id || short_names[0], + colons = `:${id}:`; + + if (custom) { + return { + id, + name, + colons, + emoticons, + custom, + imageUrl, + }; + } + + if (skin_tone) { + colons += `:skin-tone-${skin_tone}:`; + } + + return { + id, + name, + colons, + emoticons, + unified: unified.toLowerCase(), + skin: skin_tone || (skin_variations ? 1 : null), + native: unifiedToNative(unified), + }; +} + +function getSanitizedData(emoji) { + return sanitize(getData(emoji)); +} + +function getData(emoji) { + let emojiData = {}; + + if (typeof emoji === 'string') { + let matches = emoji.match(COLONS_REGEX); + + if (matches) { + emoji = matches[1]; + + } + + if (data.short_names.hasOwnProperty(emoji)) { + emoji = data.short_names[emoji]; + } + + if (data.emojis.hasOwnProperty(emoji)) { + emojiData = data.emojis[emoji]; + } + } else if (emoji.custom) { + emojiData = emoji; + + emojiData.search = buildSearch({ + short_names: emoji.short_names, + name: emoji.name, + keywords: emoji.keywords, + emoticons: emoji.emoticons, + }); + + emojiData.search = emojiData.search.join(','); + } else if (emoji.id) { + if (data.short_names.hasOwnProperty(emoji.id)) { + emoji.id = data.short_names[emoji.id]; + } + + if (data.emojis.hasOwnProperty(emoji.id)) { + emojiData = data.emojis[emoji.id]; + } + } + + emojiData.emoticons = emojiData.emoticons || []; + emojiData.variations = emojiData.variations || []; + + if (emojiData.variations && emojiData.variations.length) { + emojiData = JSON.parse(JSON.stringify(emojiData)); + emojiData.unified = emojiData.variations.shift(); + } + + return emojiData; +} + +function intersect(a, b) { + let aSet = new Set(a); + let bSet = new Set(b); + let intersection = new Set( + [...aSet].filter(x => bSet.has(x)) + ); + + return Array.from(intersection); +} + +export { getData, getSanitizedData, intersect }; diff --git a/app/javascript/mastodon/features/emoji/unicode_to_filename.js b/app/javascript/mastodon/features/emoji/unicode_to_filename.js @@ -0,0 +1,26 @@ +// taken from: +// https://github.com/twitter/twemoji/blob/47732c7/twemoji-generator.js#L848-L866 +exports.unicodeToFilename = (str) => { + let result = ''; + let charCode = 0; + let p = 0; + let i = 0; + while (i < str.length) { + charCode = str.charCodeAt(i++); + if (p) { + if (result.length > 0) { + result += '-'; + } + result += (0x10000 + ((p - 0xD800) << 10) + (charCode - 0xDC00)).toString(16); + p = 0; + } else if (0xD800 <= charCode && charCode <= 0xDBFF) { + p = charCode; + } else { + if (result.length > 0) { + result += '-'; + } + result += charCode.toString(16); + } + } + return result; +}; diff --git a/app/javascript/mastodon/features/emoji/unicode_to_unified_name.js b/app/javascript/mastodon/features/emoji/unicode_to_unified_name.js @@ -0,0 +1,17 @@ +function padLeft(str, num) { + while (str.length < num) { + str = '0' + str; + } + return str; +} + +exports.unicodeToUnifiedName = (str) => { + let output = ''; + for (let i = 0; i < str.length; i += 2) { + if (i > 0) { + output += '-'; + } + output += padLeft(str.codePointAt(i).toString(16).toUpperCase(), 4); + } + return output; +}; diff --git a/app/javascript/mastodon/reducers/accounts.js b/app/javascript/mastodon/reducers/accounts.js @@ -44,7 +44,7 @@ import { FAVOURITED_STATUSES_EXPAND_SUCCESS, } from '../actions/favourites'; import { STORE_HYDRATE } from '../actions/store'; -import emojify from '../emoji'; +import emojify from '../features/emoji/emoji'; import { Map as ImmutableMap, fromJS } from 'immutable'; import escapeTextContentForBrowser from 'escape-html'; diff --git a/app/javascript/mastodon/reducers/custom_emojis.js b/app/javascript/mastodon/reducers/custom_emojis.js @@ -1,7 +1,7 @@ import { List as ImmutableList } from 'immutable'; import { STORE_HYDRATE } from '../actions/store'; -import { search as emojiSearch } from '../emoji_index_light'; -import { buildCustomEmojis } from '../emoji'; +import { search as emojiSearch } from '../features/emoji/emoji_mart_search_light'; +import { buildCustomEmojis } from '../features/emoji/emoji'; const initialState = ImmutableList(); diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js @@ -39,7 +39,7 @@ import { PINNED_STATUSES_FETCH_SUCCESS, } from '../actions/pin_statuses'; import { SEARCH_FETCH_SUCCESS } from '../actions/search'; -import emojify from '../emoji'; +import emojify from '../features/emoji/emoji'; import { Map as ImmutableMap, fromJS } from 'immutable'; import escapeTextContentForBrowser from 'escape-html'; diff --git a/app/javascript/packs/public.js b/app/javascript/packs/public.js @@ -21,7 +21,7 @@ function main() { const { length } = require('stringz'); const IntlRelativeFormat = require('intl-relativeformat').default; const { delegate } = require('rails-ujs'); - const emojify = require('../mastodon/emoji').default; + const emojify = require('../mastodon/features/emoji/emoji').default; const { getLocale } = require('../mastodon/locales'); const { localeData } = getLocale(); const VideoContainer = require('../mastodon/containers/video_container').default; diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake @@ -17,7 +17,7 @@ namespace :emojis do task :generate do source = 'http://www.unicode.org/Public/emoji/5.0/emoji-test.txt' codes = [] - dest = Rails.root.join('app', 'javascript', 'mastodon', 'emoji_map.json') + dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json') puts "Downloading emojos from source... (#{source})" diff --git a/spec/javascript/components/emoji_index.test.js b/spec/javascript/components/emoji_index.test.js @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { search } from '../../../app/javascript/mastodon/emoji_index_light'; +import { search } from '../../../app/javascript/mastodon/features/emoji/emoji_mart_search_light'; import { emojiIndex } from 'emoji-mart'; import { pick } from 'lodash'; @@ -78,4 +78,22 @@ describe('emoji_index', () => { expect(emojiIndex.search('flag', { include: ['people'] })) .to.deep.equal([]); }); + + it('does an emoji whose unified name is irregular', () => { + let expected = [{ + 'id': 'water_polo', + 'unified': '1f93d', + 'native': '🤽', + }, { + 'id': 'man-playing-water-polo', + 'unified': '1f93d-200d-2642-fe0f', + 'native': '🤽♂️', + }, { + 'id': 'woman-playing-water-polo', + 'unified': '1f93d-200d-2640-fe0f', + 'native': '🤽♀️', + }]; + expect(search('polo').map(trimEmojis)).to.deep.equal(expected); + expect(emojiIndex.search('polo').map(trimEmojis)).to.deep.equal(expected); + }); }); diff --git a/spec/javascript/components/emojify.test.js b/spec/javascript/components/emojify.test.js @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import emojify from '../../../app/javascript/mastodon/emoji'; +import emojify from '../../../app/javascript/mastodon/features/emoji/emoji'; describe('emojify', () => { it('ignores unknown shortcodes', () => { @@ -49,4 +49,13 @@ describe('emojify', () => { expect(emojify('👌🌈💕')).to.equal('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />'); expect(emojify('👌 🌈 💕')).to.equal('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />'); }); + + it('does an emoji that has no shortcode', () => { + expect(emojify('🕉️')).to.equal('<img draggable="false" class="emojione" alt="🕉️" title="" src="/emoji/1f549.svg" />'); + }); + + it('does an emoji whose filename is irregular', () => { + expect(emojify('↙️')).to.equal('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg" />'); + }); + });