JavaScript: Stop breaking surrogate pairs in toDelta()

Resolves #69 for JavaScript

Sometimes we can find a common prefix that runs into the middle of a
surrogate pair and we split that pair when building our diff groups.

This is fine as long as we are operating on UTF-16 code units. It
becomes problematic when we start trying to treat those substrings as
valid Unicode (or UTF-8) sequences.

When we pass these split groups into `toDelta()` we do just that and the
library crashes. In this patch we're post-processing the diff groups
before encoding them to make sure that we un-split the surrogate pairs.

The post-processed diffs should produce the same output when applying
the diffs. The diff string itself will be different but should change
that much - only by a single character at surrogate boundaries.

Author: dmsnell

javascript/diff_match_patch_uncompressed.js

@@ -1339,6 +1339,15 @@ diff_match_patch.prototype.diff_levenshtein = function(diffs) {
   return levenshtein;
 };
 
+diff_match_patch.prototype.isHighSurrogate = function(c) {
+  var v = c.charCodeAt(0);
+  return v >= 0xD800 && v <= 0xDBFF;
+}
+
+diff_match_patch.prototype.isLowSurrogate = function(c) {
+  var v = c.charCodeAt(0);
+  return v >= 0xDC00 && v <= 0xDFFF;
+}
 
 /**
  * Crush the diff into an encoded string which describes the operations
@@ -1350,7 +1359,23 @@ diff_match_patch.prototype.diff_levenshtein = function(diffs) {
  */
 diff_match_patch.prototype.diff_toDelta = function(diffs) {
   var text = [];
+  var lastEnd;
   for (var x = 0; x < diffs.length; x++) {
+
+    var thisDiff = diffs[x];
+    var thisTop = thisDiff[1][0];
+    var thisEnd = thisDiff[1][thisDiff[1].length - 1];
+
+    if (thisEnd && this.isHighSurrogate(thisEnd)) {
+      thisDiff[1] = thisDiff[1].slice(0, -1);
+    }
+
+    if (lastEnd && thisTop && this.isHighSurrogate(lastEnd) && this.isLowSurrogate(thisTop)) {
+      thisDiff[1] = lastEnd + thisDiff[1];
+    }
+
+    lastEnd = thisEnd;
+
     switch (diffs[x][0]) {
       case DIFF_INSERT:
         text[x] = '+' + encodeURI(diffs[x][1]);

javascript/tests/diff_match_patch_test.js

@@ -492,6 +492,64 @@ function testDiffDelta() {
   // Convert delta string into a diff.
   assertEquivalent(diffs, dmp.diff_fromDelta(text1, delta));
 
+  diffs = [[DIFF_EQUAL, '\ud83d\ude4b\ud83d'], [DIFF_INSERT, '\ude4c\ud83d'], [DIFF_EQUAL, '\ude4b']];
+  try {
+    delta = dmp.diff_toDelta(diffs);
+    assertEquals('=2\t+%F0%9F%99%8C\t=2', delta);
+  } catch ( e ) {
+    assertEquals(false, true);
+  }
+
+  (function(){
+    const originalText = `U+1F17x	🅰️	🅱️		🅾️	🅿️ safhawifhkw
+    U+1F18x															🆎	
+    0	1	2	3	4	5	6	7	8	9	A	B	C	D	E	F
+    U+1F19x		🆑	🆒	🆓	🆔	🆕	🆖	🆗	🆘	🆙	🆚					
+    U+1F20x		🈁	🈂️							sfss.,_||saavvvbbds						
+    U+1F21x	🈚					
+    U+1F22x			🈯
+    U+1F23x			🈲	🈳	🈴	🈵	🈶	🈷️	🈸	🈹	🈺					
+    U+1F25x	🉐	🉑		
+    U+1F30x	🌀	🌁	🌂	🌃	🌄	🌅	🌆	🌇	🌈	🌉	🌊	🌋	🌌	🌍	🌎	🌏
+    U+1F31x	🌐	🌑	🌒	🌓	🌔	🌕	🌖	🌗	🌘	🌙	🌚	🌛	🌜	🌝	🌞	`;
+
+    // applies some random edits to string and returns new, edited string
+    function applyRandomTextEdit(text) {
+      let textArr = [...text];
+      let r = Math.random();
+      if(r < 1/3) { // swap
+      let swapCount = Math.floor(Math.random()*5);
+        for(let i = 0; i < swapCount; i++) {
+        let swapPos1 = Math.floor(Math.random()*textArr.length);
+          let swapPos2 = Math.floor(Math.random()*textArr.length);
+          let char1 = textArr[swapPos1];
+          let char2 = textArr[swapPos2];
+          textArr[swapPos1] = char2;
+          textArr[swapPos2] = char1;
+        }
+      } else if(r < 2/3) { // remove
+        let removeCount = Math.floor(Math.random()*5);
+        for(let i = 0; i < removeCount; i++) {
+          let removePos = Math.floor(Math.random()*textArr.length);
+          textArr[removePos] = "";
+        }
+      } else { // add
+        let addCount = Math.floor(Math.random()*5);
+        for(let i = 0; i < addCount; i++) {
+          let addPos = Math.floor(Math.random()*textArr.length);
+          let addFromPos = Math.floor(Math.random()*textArr.length);
+          textArr[addPos] = textArr[addPos] + textArr[addFromPos];
+        }
+      }
+      return textArr.join("");
+    }
+
+    for(let i = 0; i < 1000; i++) {
+      newText = applyRandomTextEdit(originalText);
+      dmp.patch_toText(dmp.patch_make(originalText, newText));
+    }
+  });
+
   // Verify pool of unchanged characters.
   diffs = [[DIFF_INSERT, 'A-Z a-z 0-9 - _ . ! ~ * \' ( ) ; / ? : @ & = + $ , # ']];
   var text2 = dmp.diff_text2(diffs);