Normalize tokens once instead of on every comparison

README.md

@@ -232,12 +232,13 @@ The simplest way to customize tokenization behavior is to simply tokenize the te
 
 To customize the notion of token equality used, use the `comparator` option to `diffArrays`.
 
-For even more customisation of the diffing behavior, you can extend the `Diff()` class, override its `castInput`, `tokenize`, `removeEmpty`, `equals`, and `join` properties with your own functions, then call its `diff(oldString, newString[, options])` method. The methods you can override are used as follows:
+For even more customisation of the diffing behavior, you can extend the `Diff()` class, override its `castInput`, `tokenize`, `removeEmpty`, `normalizeToken`, `equals`, and `join` properties with your own functions, then call its `diff(oldString, newString[, options])` method. The methods you can override are used as follows:
 
 * `castInput(value, options)`: used to transform the `oldString` and `newString` before any other steps in the diffing algorithm happen. For instance, `diffJson` uses `castInput` to serialize the objects being diffed to JSON. Defaults to a no-op.
 * `tokenize(value, options)`: used to convert each of `oldString` and `newString` (after they've gone through `castInput`) to an array of tokens. Defaults to returning `value.split('')` (returning an array of individual characters).
 * `removeEmpty(array)`: called on the arrays of tokens returned by `tokenize` and can be used to modify them. Defaults to stripping out falsey tokens, such as empty strings. `diffArrays` overrides this to simply return the `array`, which means that falsey values like empty strings can be handled like any other token by `diffArrays`.
-* `equals(left, right, options)`: called to determine if two tokens (one from the old string, one from the new string) should be considered equal. Defaults to comparing them with `===`.
+* `normalizeToken(token, options)`: called once on each token (after `removeEmpty`) to get the form it should be compared in, e.g. lowercased or trimmed. Only `equals` sees the normalized tokens; the `value`s of the returned [change objects](#change-objects) are built from the original tokens. Defaults to lowercasing string tokens if the `ignoreCase` option is set, and otherwise returning the token unchanged. Overrides should normally start from `super.normalizeToken(token, options)` so that `ignoreCase` keeps working, and must normalize each token independently of the others.
+* `equals(left, right, options)`: called to determine if two tokens (one from the old string, one from the new string) should be considered equal. Receives the tokens after they've gone through `normalizeToken`, so it does not need to apply options like `ignoreCase` itself. Defaults to calling the `comparator` option if one was passed, and otherwise comparing the tokens with `===`.
 * `join(tokens)`: gets called with an array of consecutive tokens that have either all been added, all been removed, or are all common. Needs to join them into a single value that can be used as the `value` property of the [change object](#change-objects) for these tokens. Defaults to simply returning `tokens.join('')` (and therefore by default will error out if your tokens are not strings; differs that support non-string tokens like `diffArrays` should override it to be a no-op to fix this).
 * `postProcess(changeObjects, options)`: gets called at the end of the algorithm with the [change objects](#change-objects) produced, and can do final cleanups on them. Defaults to simply returning `changeObjects` unchanged.
 

src/diff/base.ts

@@ -100,9 +100,14 @@ export default class Diff<
     const maxExecutionTime = options.timeout ?? Infinity;
     const abortAfterTimestamp = Date.now() + maxExecutionTime;
 
+    // Tokens are compared in normalized form, but the returned change objects are built from the
+    // original tokens.
+    const oldNormalized = oldTokens.map(token => this.normalizeToken(token, options)),
+          newNormalized = newTokens.map(token => this.normalizeToken(token, options));
+
     // Seed editLength = 0, i.e. the content starts with the same values
     const initialPath: Path = { oldPos: -1, lastComponent: undefined };
-    let newPos = this.extractCommon(initialPath, newTokens, oldTokens, 0, options);
+    let newPos = this.extractCommon(initialPath, newNormalized, oldNormalized, 0, options);
     if (initialPath.oldPos + 1 >= oldLen && newPos + 1 >= newLen) {
       // Identity per the equality and tokenizer
       return done(this.buildValues(initialPath.lastComponent, newTokens, oldTokens));
@@ -186,7 +191,7 @@ export default class Diff<
           basePath = this.addToPath(removePath, false, true, 1, options);
         }
 
-        newPos = this.extractCommon(basePath, newTokens, oldTokens, k, options);
+        newPos = this.extractCommon(basePath, newNormalized, oldNormalized, k, options);
 
         if (basePath.oldPos + 1 >= oldLen && newPos + 1 >= newLen) {
           // If we have hit the end of both strings, then we are done
@@ -282,13 +287,33 @@ export default class Diff<
     return newPos;
   }
 
+  // Receives tokens that have already been through normalizeToken().
   equals(left: TokenT, right: TokenT, options: AllDiffOptions): boolean {
     if (options.comparator) {
       return options.comparator(left, right);
     } else {
-      return left === right
-        || (!!options.ignoreCase && (left as string).toLowerCase() === (right as string).toLowerCase());
+      return left === right;
+    }
+  }
+
+  // Maps a token to the form it should be compared in (e.g. lowercased). Called once per token up
+  // front, rather than on every call to equals(), which the diff loop makes many times per token.
+  // Subclasses that override this should build on super.normalizeToken() so that ignoreCase keeps
+  // working.
+  normalizeToken(token: TokenT, options: AllDiffOptions): TokenT {
+    // Only strings have a case. Tokens can be any value in diffArrays, which has no ignoreCase
+    // option but may still be passed one, so leave other tokens alone rather than throwing.
+    if (options.ignoreCase && typeof token === 'string') {
+      // toLowerCase() may return a copy even when nothing changed (e.g. in V8 for longer or
+      // non-ASCII strings), and the normalized tokens are kept for the whole diff. Returning the
+      // original token where we can means only tokens that contained uppercase letters cost
+      // extra memory.
+      const lower = token.toLowerCase();
+      if(lower !== token) {
+        return lower as TokenT;
+      }
     }
+    return token;
   }
 
   removeEmpty(array: TokenT[]): TokenT[] {

src/diff/json.ts

@@ -17,8 +17,8 @@ class JsonDiff extends Diff<string, string, string | object> {
     return typeof value === 'string' ? value : JSON.stringify(canonicalize(value, null, null, stringifyReplacer), null, '  ');
   }
 
-  equals(left: string, right: string, options: DiffJsonOptionsNonabortable | DiffJsonOptionsAbortable) {
-    return super.equals(left.replace(/,([\r\n])/g, '$1'), right.replace(/,([\r\n])/g, '$1'), options);
+  normalizeToken(token: string, options: DiffJsonOptionsNonabortable | DiffJsonOptionsAbortable) {
+    return super.normalizeToken(token, options).replace(/,([\r\n])/g, '$1');
   }
 }
 

src/diff/line.ts

@@ -5,7 +5,9 @@ import {generateOptions} from '../util/params.js';
 class LineDiff extends Diff<string, string> {
   tokenize = tokenize;
 
-  equals(left: string, right: string, options: DiffLinesOptionsAbortable | DiffLinesOptionsNonabortable) {
+  normalizeToken(token: string, options: DiffLinesOptionsAbortable | DiffLinesOptionsNonabortable) {
+    // Casing is normalized in super
+    token = super.normalizeToken(token, options);
     // If we're ignoring whitespace, we need to normalise lines by stripping
     // whitespace before checking equality. (This has an annoying interaction
     // with newlineIsToken that requires special handling: if newlines get their
@@ -14,21 +16,15 @@ class LineDiff extends Diff<string, string> {
     // as equal to a separator between lines, which would be weird and
     // inconsistent with the documented behavior of the options.)
     if (options.ignoreWhitespace) {
-      if (!options.newlineIsToken || !left.includes('\n')) {
-        left = left.trim();
-      }
-      if (!options.newlineIsToken || !right.includes('\n')) {
-        right = right.trim();
+      if (!options.newlineIsToken || !token.includes('\n')) {
+        token = token.trim();
       }
     } else if (options.ignoreNewlineAtEof && !options.newlineIsToken) {
-      if (left.endsWith('\n')) {
-        left = left.slice(0, -1);
-      }
-      if (right.endsWith('\n')) {
-        right = right.slice(0, -1);
+      if (token.endsWith('\n')) {
+        token = token.slice(0, -1);
       }
     }
-    return super.equals(left, right, options);
+    return token;
   }
 }
 

src/diff/word.ts

@@ -42,8 +42,8 @@ const extendedWordChars = 'a-zA-Z0-9_\\u{AD}\\u{C0}-\\u{D6}\\u{D8}-\\u{F6}\\u{F8
 //   See https://github.com/kpdecker/jsdiff/issues/160#issuecomment-1866099640
 //   for an example.
 //
-// Keeping the surrounding whitespace of course has implications for .equals
-// and .join, not just .tokenize.
+// Keeping the surrounding whitespace of course has implications for
+// .normalizeToken and .join, not just .tokenize.
 
 // This regex does NOT fully implement the tokenization rules described above.
 // Instead, it gives runs of whitespace their own "token". The tokenize method
@@ -53,13 +53,11 @@ const tokenizeIncludingWhitespace = new RegExp(`[${extendedWordChars}]+|\\s+|[^$
 
 
 class WordDiff extends Diff<string, string> {
-  equals(left: string, right: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable) {
-    if (options.ignoreCase) {
-      left = left.toLowerCase();
-      right = right.toLowerCase();
-    }
-
-    return left.trim() === right.trim();
+  normalizeToken(token: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable) {
+    return super.normalizeToken(token, options)
+      // Unlike the other normalizers, this trim doesn't depend on any option: diffWords always
+      // ignores the whitespace around words, which every token carries (see above).
+      .trim();
   }
 
   tokenize(value: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable = {}) {

test/diff/array.js

@@ -61,6 +61,17 @@ describe('diff/array', function() {
         expect(diffResult2[0].value).to.not.equal(input2);
       });
     });
+    it('Should not throw on non-string items when passed ignoreCase', function() {
+      const a = {a: 0}, b = {a: 1}, c = {a: 2};
+      expect(diffArrays([a, b], [a, b], { ignoreCase: true })).to.deep.equals([
+          {count: 2, value: [a, b], removed: false, added: false}
+      ]);
+      expect(diffArrays([a, b], [a, c], { ignoreCase: true })).to.deep.equals([
+          {count: 1, value: [a], removed: false, added: false},
+          {count: 1, value: [b], removed: true, added: false},
+          {count: 1, value: [c], removed: false, added: true}
+      ]);
+    });
     it('Should diff arrays with comparator', function() {
       const a = {a: 0}, b = {a: 1}, c = {a: 2}, d = {a: 3};
       function comparator(left, right) {

test/diff/character.js

@@ -69,6 +69,15 @@ describe('diff/character', function() {
         const diffResult = diffChars('New Values.', 'New value.', {ignoreCase: true});
         expect(convertChangesToXML(diffResult)).to.equal('New value<del>s</del>.');
       });
+
+      it('is applied before tokens are passed to a comparator', function() {
+        const seen = [];
+        diffChars('A', 'b', {ignoreCase: true, comparator: (left, right) => {
+          seen.push(left + right);
+          return left === right;
+        }});
+        expect(seen).to.eql(['ab']);
+      });
     });
 
     it('should not be susceptible to race conditions in async mode when called with different options', function(done) {