Tokenize lines with indexOf and slice instead of a regex split

src/diff/line.ts

@@ -105,28 +105,35 @@ export function diffTrimmedLines(oldStr: string, newStr: string, options?: any):
 
 // Exported standalone so it can be used from jsonDiff too.
 export function tokenize(value: string, options: DiffLinesOptionsAbortable | DiffLinesOptionsNonabortable) {
+  if (typeof value !== 'string' && !((value as unknown) instanceof String)) {
+    // Without this, non-string input would silently be diffed as if it were empty
+    throw new TypeError(`Expected a string to split into lines, got ${typeof value}`);
+  }
   if(options.stripTrailingCr) {
     // remove one \r before \n to match GNU diff's --strip-trailing-cr behavior
     value = value.replace(/\r\n/g, '\n');
   }
 
-  const retLines = [],
-      linesAndNewlines = value.split(/(\n|\r\n)/);
-
-  // Ignore the final empty token that occurs if the string ends with a new line
-  if (!linesAndNewlines[linesAndNewlines.length - 1]) {
-    linesAndNewlines.pop();
-  }
+  const retLines = [];
+  let lineStart = 0;
 
-  // Merge the content and line separators into single tokens
-  for (let i = 0; i < linesAndNewlines.length; i++) {
-    const line = linesAndNewlines[i];
+  while (lineStart < value.length) {
+    const newlineIndex = value.indexOf('\n', lineStart);
+    if (newlineIndex === -1) {
+      // The last line has no line break
+      retLines.push(value.slice(lineStart));
+      break;
+    }
 
-    if (i % 2 && !options.newlineIsToken) {
-      retLines[retLines.length - 1] += line;
+    const lineEnd = newlineIndex + 1;
+    if (options.newlineIsToken) {
+      // The line break (\n or \r\n) is a token of its own
+      const lineBreak = value.endsWith('\r\n', lineEnd) ? '\r\n' : '\n';
+      retLines.push(value.slice(lineStart, lineEnd - lineBreak.length), lineBreak);
     } else {
-      retLines.push(line);
+      retLines.push(value.slice(lineStart, lineEnd));
     }
+    lineStart = lineEnd;
   }
 
   return retLines;

test/diff/line.js

@@ -1,4 +1,4 @@
-import {diffLines, diffTrimmedLines} from '../../libesm/diff/line.js';
+import {diffLines, diffTrimmedLines, tokenize} from '../../libesm/diff/line.js';
 import {convertChangesToXML} from '../../libesm/convert/xml.js';
 
 import {expect} from 'chai';
@@ -47,6 +47,16 @@ describe('diff/line', function() {
       expect(convertChangesToXML(diffResult)).to.equal('<del>line\n\nold value \n\nline</del>');
     });
 
+    it('should tokenize each line together with its line break', function() {
+      expect(tokenize('', {})).to.deep.equal([]);
+      expect(tokenize('\n\n', {})).to.deep.equal(['\n', '\n']);
+      expect(tokenize('a\nb', {})).to.deep.equal(['a\n', 'b']);
+      expect(tokenize('a\r\nb\r\n', {})).to.deep.equal(['a\r\n', 'b\r\n']);
+      // A lone \r is not a line break
+      expect(tokenize('a\rb\n', {})).to.deep.equal(['a\rb\n']);
+      expect(tokenize('a\r\n\nb', {newlineIsToken: true})).to.deep.equal(['a', '\r\n', '', '\n', 'b']);
+    });
+
     it('Should prefer to do deletions before insertions, like Unix diff does', function() {
       const diffResult = diffLines('a\nb\nc\nd\n', 'a\nc\nb\nd\n');