diff --git a/README.md b/README.md index 2ff57623..a05d7739 100644 --- a/README.md +++ b/README.md @@ -232,12 +232,13 @@ The simplest way to customize tokenization behavior is to simply tokenize the te To customize the notion of token equality used, use the `comparator` option to `diffArrays`. -For even more customisation of the diffing behavior, you can extend the `Diff()` class, override its `castInput`, `tokenize`, `removeEmpty`, `equals`, and `join` properties with your own functions, then call its `diff(oldString, newString[, options])` method. The methods you can override are used as follows: +For even more customisation of the diffing behavior, you can extend the `Diff()` class, override its `castInput`, `tokenize`, `removeEmpty`, `normalizeToken`, `equals`, and `join` properties with your own functions, then call its `diff(oldString, newString[, options])` method. The methods you can override are used as follows: * `castInput(value, options)`: used to transform the `oldString` and `newString` before any other steps in the diffing algorithm happen. For instance, `diffJson` uses `castInput` to serialize the objects being diffed to JSON. Defaults to a no-op. * `tokenize(value, options)`: used to convert each of `oldString` and `newString` (after they've gone through `castInput`) to an array of tokens. Defaults to returning `value.split('')` (returning an array of individual characters). * `removeEmpty(array)`: called on the arrays of tokens returned by `tokenize` and can be used to modify them. Defaults to stripping out falsey tokens, such as empty strings. `diffArrays` overrides this to simply return the `array`, which means that falsey values like empty strings can be handled like any other token by `diffArrays`. -* `equals(left, right, options)`: called to determine if two tokens (one from the old string, one from the new string) should be considered equal. Defaults to comparing them with `===`. +* `normalizeToken(token, options)`: called once on each token (after `removeEmpty`) to get the form it should be compared in, e.g. lowercased or trimmed. Only `equals` sees the normalized tokens; the `value`s of the returned [change objects](#change-objects) are built from the original tokens. Defaults to lowercasing string tokens if the `ignoreCase` option is set, and otherwise returning the token unchanged. Overrides should normally start from `super.normalizeToken(token, options)` so that `ignoreCase` keeps working, and must normalize each token independently of the others. +* `equals(left, right, options)`: called to determine if two tokens (one from the old string, one from the new string) should be considered equal. Receives the tokens after they've gone through `normalizeToken`, so it does not need to apply options like `ignoreCase` itself. Defaults to calling the `comparator` option if one was passed, and otherwise comparing the tokens with `===`. * `join(tokens)`: gets called with an array of consecutive tokens that have either all been added, all been removed, or are all common. Needs to join them into a single value that can be used as the `value` property of the [change object](#change-objects) for these tokens. Defaults to simply returning `tokens.join('')` (and therefore by default will error out if your tokens are not strings; differs that support non-string tokens like `diffArrays` should override it to be a no-op to fix this). * `postProcess(changeObjects, options)`: gets called at the end of the algorithm with the [change objects](#change-objects) produced, and can do final cleanups on them. Defaults to simply returning `changeObjects` unchanged. diff --git a/src/diff/base.ts b/src/diff/base.ts index 0da25ba4..a4f4409a 100644 --- a/src/diff/base.ts +++ b/src/diff/base.ts @@ -100,9 +100,14 @@ export default class Diff< const maxExecutionTime = options.timeout ?? Infinity; const abortAfterTimestamp = Date.now() + maxExecutionTime; + // Tokens are compared in normalized form, but the returned change objects are built from the + // original tokens. + const oldNormalized = oldTokens.map(token => this.normalizeToken(token, options)), + newNormalized = newTokens.map(token => this.normalizeToken(token, options)); + // Seed editLength = 0, i.e. the content starts with the same values const initialPath: Path = { oldPos: -1, lastComponent: undefined }; - let newPos = this.extractCommon(initialPath, newTokens, oldTokens, 0, options); + let newPos = this.extractCommon(initialPath, newNormalized, oldNormalized, 0, options); if (initialPath.oldPos + 1 >= oldLen && newPos + 1 >= newLen) { // Identity per the equality and tokenizer return done(this.buildValues(initialPath.lastComponent, newTokens, oldTokens)); @@ -186,7 +191,7 @@ export default class Diff< basePath = this.addToPath(removePath, false, true, 1, options); } - newPos = this.extractCommon(basePath, newTokens, oldTokens, k, options); + newPos = this.extractCommon(basePath, newNormalized, oldNormalized, k, options); if (basePath.oldPos + 1 >= oldLen && newPos + 1 >= newLen) { // If we have hit the end of both strings, then we are done @@ -282,13 +287,33 @@ export default class Diff< return newPos; } + // Receives tokens that have already been through normalizeToken(). equals(left: TokenT, right: TokenT, options: AllDiffOptions): boolean { if (options.comparator) { return options.comparator(left, right); } else { - return left === right - || (!!options.ignoreCase && (left as string).toLowerCase() === (right as string).toLowerCase()); + return left === right; + } + } + + // Maps a token to the form it should be compared in (e.g. lowercased). Called once per token up + // front, rather than on every call to equals(), which the diff loop makes many times per token. + // Subclasses that override this should build on super.normalizeToken() so that ignoreCase keeps + // working. + normalizeToken(token: TokenT, options: AllDiffOptions): TokenT { + // Only strings have a case. Tokens can be any value in diffArrays, which has no ignoreCase + // option but may still be passed one, so leave other tokens alone rather than throwing. + if (options.ignoreCase && typeof token === 'string') { + // toLowerCase() may return a copy even when nothing changed (e.g. in V8 for longer or + // non-ASCII strings), and the normalized tokens are kept for the whole diff. Returning the + // original token where we can means only tokens that contained uppercase letters cost + // extra memory. + const lower = token.toLowerCase(); + if(lower !== token) { + return lower as TokenT; + } } + return token; } removeEmpty(array: TokenT[]): TokenT[] { diff --git a/src/diff/json.ts b/src/diff/json.ts index 622ba917..102c00bd 100644 --- a/src/diff/json.ts +++ b/src/diff/json.ts @@ -17,8 +17,8 @@ class JsonDiff extends Diff { return typeof value === 'string' ? value : JSON.stringify(canonicalize(value, null, null, stringifyReplacer), null, ' '); } - equals(left: string, right: string, options: DiffJsonOptionsNonabortable | DiffJsonOptionsAbortable) { - return super.equals(left.replace(/,([\r\n])/g, '$1'), right.replace(/,([\r\n])/g, '$1'), options); + normalizeToken(token: string, options: DiffJsonOptionsNonabortable | DiffJsonOptionsAbortable) { + return super.normalizeToken(token, options).replace(/,([\r\n])/g, '$1'); } } diff --git a/src/diff/line.ts b/src/diff/line.ts index 53de7c21..c0479e94 100644 --- a/src/diff/line.ts +++ b/src/diff/line.ts @@ -5,7 +5,9 @@ import {generateOptions} from '../util/params.js'; class LineDiff extends Diff { tokenize = tokenize; - equals(left: string, right: string, options: DiffLinesOptionsAbortable | DiffLinesOptionsNonabortable) { + normalizeToken(token: string, options: DiffLinesOptionsAbortable | DiffLinesOptionsNonabortable) { + // Casing is normalized in super + token = super.normalizeToken(token, options); // If we're ignoring whitespace, we need to normalise lines by stripping // whitespace before checking equality. (This has an annoying interaction // with newlineIsToken that requires special handling: if newlines get their @@ -14,21 +16,15 @@ class LineDiff extends Diff { // as equal to a separator between lines, which would be weird and // inconsistent with the documented behavior of the options.) if (options.ignoreWhitespace) { - if (!options.newlineIsToken || !left.includes('\n')) { - left = left.trim(); - } - if (!options.newlineIsToken || !right.includes('\n')) { - right = right.trim(); + if (!options.newlineIsToken || !token.includes('\n')) { + token = token.trim(); } } else if (options.ignoreNewlineAtEof && !options.newlineIsToken) { - if (left.endsWith('\n')) { - left = left.slice(0, -1); - } - if (right.endsWith('\n')) { - right = right.slice(0, -1); + if (token.endsWith('\n')) { + token = token.slice(0, -1); } } - return super.equals(left, right, options); + return token; } } diff --git a/src/diff/word.ts b/src/diff/word.ts index 81606a31..0262d50b 100644 --- a/src/diff/word.ts +++ b/src/diff/word.ts @@ -42,8 +42,8 @@ const extendedWordChars = 'a-zA-Z0-9_\\u{AD}\\u{C0}-\\u{D6}\\u{D8}-\\u{F6}\\u{F8 // See https://github.com/kpdecker/jsdiff/issues/160#issuecomment-1866099640 // for an example. // -// Keeping the surrounding whitespace of course has implications for .equals -// and .join, not just .tokenize. +// Keeping the surrounding whitespace of course has implications for +// .normalizeToken and .join, not just .tokenize. // This regex does NOT fully implement the tokenization rules described above. // Instead, it gives runs of whitespace their own "token". The tokenize method @@ -53,13 +53,11 @@ const tokenizeIncludingWhitespace = new RegExp(`[${extendedWordChars}]+|\\s+|[^$ class WordDiff extends Diff { - equals(left: string, right: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable) { - if (options.ignoreCase) { - left = left.toLowerCase(); - right = right.toLowerCase(); - } - - return left.trim() === right.trim(); + normalizeToken(token: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable) { + return super.normalizeToken(token, options) + // Unlike the other normalizers, this trim doesn't depend on any option: diffWords always + // ignores the whitespace around words, which every token carries (see above). + .trim(); } tokenize(value: string, options: DiffWordsOptionsAbortable | DiffWordsOptionsNonabortable = {}) { diff --git a/test/diff/array.js b/test/diff/array.js index 35c56ba8..f955c1a6 100644 --- a/test/diff/array.js +++ b/test/diff/array.js @@ -61,6 +61,17 @@ describe('diff/array', function() { expect(diffResult2[0].value).to.not.equal(input2); }); }); + it('Should not throw on non-string items when passed ignoreCase', function() { + const a = {a: 0}, b = {a: 1}, c = {a: 2}; + expect(diffArrays([a, b], [a, b], { ignoreCase: true })).to.deep.equals([ + {count: 2, value: [a, b], removed: false, added: false} + ]); + expect(diffArrays([a, b], [a, c], { ignoreCase: true })).to.deep.equals([ + {count: 1, value: [a], removed: false, added: false}, + {count: 1, value: [b], removed: true, added: false}, + {count: 1, value: [c], removed: false, added: true} + ]); + }); it('Should diff arrays with comparator', function() { const a = {a: 0}, b = {a: 1}, c = {a: 2}, d = {a: 3}; function comparator(left, right) { diff --git a/test/diff/character.js b/test/diff/character.js index 42d78aa1..f4271666 100644 --- a/test/diff/character.js +++ b/test/diff/character.js @@ -69,6 +69,15 @@ describe('diff/character', function() { const diffResult = diffChars('New Values.', 'New value.', {ignoreCase: true}); expect(convertChangesToXML(diffResult)).to.equal('New values.'); }); + + it('is applied before tokens are passed to a comparator', function() { + const seen = []; + diffChars('A', 'b', {ignoreCase: true, comparator: (left, right) => { + seen.push(left + right); + return left === right; + }}); + expect(seen).to.eql(['ab']); + }); }); it('should not be susceptible to race conditions in async mode when called with different options', function(done) {