[WIP] MUCH better approach to replacing emojis with still versions

.babelrc

@@ -1,5 +1,5 @@
 {
-  "presets": ["@babel/preset-env"],
+  "presets": ["@babel/preset-env", "@vue/babel-preset-jsx"],
-  "plugins": ["@babel/plugin-transform-runtime", "lodash", "@vue/babel-plugin-transform-vue-jsx"],
+  "plugins": ["@babel/plugin-transform-runtime", "lodash"],
   "comments": false
 }

package.json

@@ -47,8 +47,8 @@
     "@babel/preset-env": "^7.7.6",
     "@babel/register": "^7.7.4",
     "@ungap/event-target": "^0.1.0",
-    "@vue/babel-helper-vue-jsx-merge-props": "^1.0.0",
+    "@vue/babel-helper-vue-jsx-merge-props": "^1.2.1",
-    "@vue/babel-plugin-transform-vue-jsx": "^1.1.2",
+    "@vue/babel-preset-jsx": "^1.2.4",
     "@vue/test-utils": "^1.0.0-beta.26",
     "autoprefixer": "^6.4.0",
     "babel-eslint": "^7.0.0",

src/components/rich_content/rich_content.jsx

@@ -0,0 +1,66 @@
+import Vue from 'vue'
+import { mapGetters } from 'vuex'
+import { processHtml } from 'src/services/tiny_post_html_processor/tiny_post_html_processor.service.js'
+import { convertHtml, getTagName, processTextForEmoji, getAttrs } from 'src/services/mini_html_converter/mini_html_converter.service.js'
+import { mentionMatchesUrl, extractTagFromUrl } from 'src/services/matcher/matcher.service.js'
+import generateProfileLink from 'src/services/user_profile_link_generator/user_profile_link_generator'
+import StillImage from 'src/components/still-image/still-image.vue'
+
+import './rich_content.scss'
+
+export default Vue.component('RichContent', {
+  name: 'RichContent',
+  props: {
+    html: {
+      required: true,
+      type: String
+    },
+    emoji: {
+      required: true,
+      type: Array
+    }
+  },
+  render (h) {
+    const renderImage = (tag) => {
+      return <StillImage {...{ attrs: getAttrs(tag) }} />
+    }
+    const structure = convertHtml(this.html)
+    const processItem = (item) => {
+      if (typeof item === 'string') {
+        if (item.includes(':')) {
+          return processTextForEmoji(
+            item,
+            this.emoji,
+            ({ shortcode, url }) => {
+              return <StillImage
+                class="emoji"
+                src={url}
+                title={`:${shortcode}:`}
+                alt={`:${shortcode}:`}
+              />
+            }
+          )
+        } else {
+          return item
+        }
+      }
+      if (Array.isArray(item)) {
+        const [opener, children] = item
+        const Tag = getTagName(opener)
+        if (Tag === 'img') {
+          return renderImage(opener)
+        }
+        if (children !== undefined) {
+          return <Tag {...{ attrs: getAttrs(opener) }}>
+            { children.map(processItem) }
+          </Tag>
+        } else {
+          return <Tag/>
+        }
+      }
+    }
+    return <div>
+      { structure.map(processItem) }
+    </div>
+  }
+})

src/components/status_content/status_content.js

@@ -1,6 +1,7 @@
 import Attachment from '../attachment/attachment.vue'
 import Poll from '../poll/poll.vue'
 import Gallery from '../gallery/gallery.vue'
+import RichContent from 'src/components/rich_content/rich_content.jsx'
 import LinkPreview from '../link-preview/link-preview.vue'
 import generateProfileLink from 'src/services/user_profile_link_generator/user_profile_link_generator'
 import fileType from 'src/services/file_type/file_type.service'
@@ -125,7 +126,7 @@ const StatusContent = {
       return this.mergedConfig.maxThumbnails
     },
     postBodyHtml () {
-      const html = this.status.statusnet_html
+      const html = this.status.raw_html
 
       if (this.mergedConfig.greentext) {
         try {
@@ -164,7 +165,8 @@ const StatusContent = {
     Attachment,
     Poll,
     Gallery,
-    LinkPreview
+    LinkPreview,
+    RichContent
   },
   methods: {
     linkClicked (event) {

src/components/status_content/status_content.vue

@@ -1,5 +1,4 @@
 <template>
-  <!-- eslint-disable vue/no-v-html -->
   <div class="StatusContent">
     <slot name="header" />
     <div
@@ -7,11 +6,11 @@
       class="summary-wrapper"
       :class="{ 'tall-subject': (longSubject && !showingLongSubject) }"
     >
-      <div
+      <RichContent
         class="media-body summary"
         @click.prevent="linkClicked"
-        v-html="status.summary_html"
+        :html="status.summary_raw_html"
-      />
+        :emoji="status.emojis"/>
       <button
         v-if="longSubject && showingLongSubject"
         class="button-unstyled -link tall-subject-hider"
@@ -40,13 +39,13 @@
       >
         {{ $t("general.show_more") }}
       </button>
-      <div
+      <RichContent
         v-if="!hideSubjectStatus"
         :class="{ 'single-line': singleLine }"
         class="status-content media-body"
         @click.prevent="linkClicked"
-        v-html="postBodyHtml"
+        :html="postBodyHtml"
-      />
+        :emoji="status.emojis"/>
       <button
         v-if="hideSubjectStatus"
         class="button-unstyled -link cw-status-hider"
@@ -127,7 +126,6 @@
     </div>
     <slot name="footer" />
   </div>
-  <!-- eslint-enable vue/no-v-html -->
 </template>
 
 <script src="./status_content.js" ></script>

src/services/entity_normalizer/entity_normalizer.service.js

@@ -267,6 +267,8 @@ export const parseStatus = (data) => {
     output.nsfw = data.sensitive
 
     output.statusnet_html = addEmojis(data.content, data.emojis)
+    output.raw_html = data.content
+    output.emojis = data.emojis
 
     output.tags = data.tags
 
@@ -293,6 +295,7 @@ export const parseStatus = (data) => {
       output.retweeted_status = parseStatus(data.reblog)
     }
 
+    output.summary_raw_html = escape(data.spoiler_text)
     output.summary_html = addEmojis(escape(data.spoiler_text), data.emojis)
     output.external_url = data.url
     output.poll = data.poll

src/services/mini_html_converter/mini_html_converter.service.js

@@ -0,0 +1,137 @@
+/**
+ * This is a not-so-tiny purpose-built HTML parser/processor. It was made for use
+ * with StatusText component for purpose of replacing tags with vue components
+ *
+ * known issue: doesn't handle CDATA so nested CDATA might not work well
+ *
+ * @param {Object} input - input data
+ * @param {(string) => string} lineProcessor - function that will be called on every line
+ * @param {{ key[string]: (string) => string}} tagProcessor - map of processors for tags
+ * @return {string} processed html
+ */
+export const convertHtml = (html) => {
+  // Elements that are implicitly self-closing
+  // https://developer.mozilla.org/en-US/docs/Glossary/empty_element
+  const emptyElements = new Set([
+    'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
+    'keygen', 'link', 'meta', 'param', 'source', 'track', 'wbr'
+  ])
+  // TODO For future - also parse HTML5 multi-source components?
+
+  const buffer = [] // Current output buffer
+  const levels = [['', buffer]] // How deep we are in tags and which tags were there
+  let textBuffer = '' // Current line content
+  let tagBuffer = null // Current tag buffer, if null = we are not currently reading a tag
+
+  const getCurrentBuffer = () => {
+    return levels[levels.length - 1][1]
+  }
+
+  const flushText = () => { // Processes current line buffer, adds it to output buffer and clears line buffer
+    if (textBuffer === '') return
+    getCurrentBuffer().push(textBuffer)
+    textBuffer = ''
+  }
+
+  const handleSelfClosing = (tag) => {
+    getCurrentBuffer().push([tag])
+  }
+
+  const handleOpen = (tag) => {
+    const curBuf = getCurrentBuffer()
+    const newLevel = [tag, []]
+    levels.push(newLevel)
+    curBuf.push(newLevel)
+  }
+
+  const handleClose = (tag) => {
+    const currentTag = levels[levels.length - 1]
+    if (getTagName(levels[levels.length - 1][0]) === getTagName(tag)) {
+      currentTag.push(tag)
+      levels.pop()
+    } else {
+      getCurrentBuffer().push(tag)
+    }
+  }
+
+  for (let i = 0; i < html.length; i++) {
+    const char = html[i]
+    if (char === '<' && tagBuffer === null) {
+      flushText()
+      tagBuffer = char
+    } else if (char !== '>' && tagBuffer !== null) {
+      tagBuffer += char
+    } else if (char === '>' && tagBuffer !== null) {
+      tagBuffer += char
+      const tagFull = tagBuffer
+      tagBuffer = null
+      const tagName = getTagName(tagFull)
+      if (tagFull[1] === '/') {
+        handleClose(tagFull)
+      } else if (emptyElements.has(tagName) || tagFull[tagFull.length - 2] === '/') {
+        // self-closing
+        handleSelfClosing(tagFull)
+      } else {
+        handleOpen(tagFull)
+      }
+    } else {
+      textBuffer += char
+    }
+  }
+  if (tagBuffer) {
+    textBuffer += tagBuffer
+  }
+
+  flushText()
+  return buffer
+}
+
+// Extracts tag name from tag, i.e. <span a="b"> => span
+export const getTagName = (tag) => {
+  const result = /(?:<\/(\w+)>|<(\w+)\s?.*?\/?>)/gi.exec(tag)
+  return result && (result[1] || result[2])
+}
+
+export const processTextForEmoji = (text, emojis, processor) => {
+  const buffer = []
+  let textBuffer = ''
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i]
+    if (char === ':') {
+      const next = text.slice(i + 1)
+      let found = false
+      for (let emoji of emojis) {
+        if (next.slice(0, emoji.shortcode.length + 1) === (emoji.shortcode + ':')) {
+          found = emoji
+          break
+        }
+      }
+      if (found) {
+        buffer.push(textBuffer)
+        textBuffer = ''
+        buffer.push(processor(found))
+        i += found.shortcode.length + 1
+      } else {
+        textBuffer += char
+      }
+    } else {
+      textBuffer += char
+    }
+  }
+  return buffer
+}
+
+export const getAttrs = tag => {
+  const innertag = tag
+    .substring(1, tag.length - 1)
+    .replace(new RegExp('^' + getTagName(tag)), '')
+    .replace(/\/?$/, '')
+    .trim()
+  const attrs = Array.from(innertag.matchAll(/([a-z0-9-]+)(?:=(?:"([^"]+?)"|'([^']+?)'))?/gi))
+    .map(([trash, key, value]) => [key, value])
+    .map(([k, v]) => {
+      if (!v) return [k, true]
+      return [k, v]
+    })
+  return Object.fromEntries(attrs)
+}

test/unit/specs/services/tiny_post_html_processor/mini_post_html_processor.spec.js

@@ -0,0 +1,130 @@
+import { convertHtml, processTextForEmoji } from 'src/services/mini_html_converter/mini_html_converter.service.js'
+
+describe('MiniHtmlConverter', () => {
+  describe('convertHtml', () => {
+    it('converts html into a tree structure', () => {
+      const inputOutput = '1 <p>2</p> <b>3<img src="a">4</b>5'
+      expect(convertHtml(inputOutput)).to.eql([
+        '1 ',
+        [
+          '<p>',
+          ['2'],
+          '</p>'
+        ],
+        ' ',
+        [
+          '<b>',
+          [
+            '3',
+            ['<img src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts html to tree while preserving tag formatting', () => {
+      const inputOutput = '1 <p >2</p><b >3<img   src="a">4</b>5'
+      expect(convertHtml(inputOutput)).to.eql([
+        '1 ',
+        [
+          '<p >',
+          ['2'],
+          '</p>'
+        ],
+        [
+          '<b >',
+          [
+            '3',
+            ['<img   src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts semi-broken html', () => {
+      const inputOutput = '1 <br> 2 <p> 42'
+      expect(convertHtml(inputOutput)).to.eql([
+        '1 ',
+        ['<br>'],
+        ' 2 ',
+        [
+          '<p>',
+          [' 42']
+        ]
+      ])
+    })
+    it('realistic case', () => {
+      const inputOutput = '<p><span class="h-card"><a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">@<span>benis</span></a></span> <span class="h-card"><a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">@<span>hj</span></a></span> nice</p>'
+      expect(convertHtml(inputOutput)).to.eql([
+        [
+          '<p>',
+          [
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'benis'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' ',
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'hj'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' nice'
+          ],
+          '</p>'
+        ]
+      ])
+    })
+  })
+  describe('processTextForEmoji', () => {
+    it('processes all emoji in text', () => {
+      const inputOutput = 'Hello from finland! :lol: We have best water! :lmao:'
+      const emojis = [
+        { shortcode: 'lol', src: 'LOL' },
+        { shortcode: 'lmao', src: 'LMAO' }
+      ]
+      const processor = ({ shortcode, src }) => ({ shortcode, src })
+      expect(processTextForEmoji(inputOutput, emojis, processor)).to.eql([
+        'Hello from finland! ',
+        { shortcode: 'lol', src: 'LOL' },
+        ' We have best water! ',
+        { shortcode: 'lmao', src: 'LMAO' }
+      ])
+    })
+  })
+})