diff --git a/dist/commonjs/atomic.d.ts b/dist/commonjs/atomic.d.ts index 6807242..deb10e9 100644 --- a/dist/commonjs/atomic.d.ts +++ b/dist/commonjs/atomic.d.ts @@ -1,6 +1,6 @@ export interface StringComparisonOptions { - transforms?: ((str: string) => string)[]; - strategies?: ComparisonStrategy[]; + transforms?: StringTransformFunc[]; + strategies?: ComparisonStrategy[]; } export interface StringSamenessResult { strategies: { @@ -26,8 +26,8 @@ export interface NamedComparisonStrategyObjectResult extends ComparisonStrategyR name: string; } export type ComparisonStrategyResultValue = number | ComparisonStrategyResultObject; -export type StrategyFunc = (strA: string, strB: string) => ComparisonStrategyResultValue; -export interface ComparisonStrategy { +export type StrategyFunc = (strA: string, strB: string) => T; +export interface ComparisonStrategy { /** * The name of this strategy * */ @@ -35,9 +35,10 @@ export interface ComparisonStrategy { /** * A function that accepts two string arguments and returns a number * */ - strategy: StrategyFunc; + strategy: StrategyFunc; /** * An optional function that accepts two string arguments and returns whether this strategy should be used * */ isValid?: (strA: string, strB: string) => boolean; } +export type StringTransformFunc = (str: string) => string; diff --git a/dist/commonjs/atomic.js b/dist/commonjs/atomic.js index c8ad2e5..25299c0 100644 --- a/dist/commonjs/atomic.js +++ b/dist/commonjs/atomic.js @@ -1,2 +1,3 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); +//# sourceMappingURL=atomic.js.map \ No newline at end of file diff --git a/dist/commonjs/atomic.js.map b/dist/commonjs/atomic.js.map new file mode 100644 index 0000000..d76053d --- /dev/null +++ b/dist/commonjs/atomic.js.map @@ -0,0 +1 @@ +{"version":3,"file":"atomic.js","sourceRoot":"","sources":["../../src/atomic.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/commonjs/index.d.ts b/dist/commonjs/index.d.ts index 8c1309f..a9ac7c2 100644 --- a/dist/commonjs/index.d.ts +++ b/dist/commonjs/index.d.ts @@ -1,11 +1,13 @@ -import { StringComparisonOptions, StringSamenessResult } from "./atomic.js"; -declare const defaultStrCompareTransformFuncs: ((str: string) => string)[]; -declare const defaultStrategies: import("./atomic.js").ComparisonStrategy[]; +import { ComparisonStrategyResult, StringComparisonOptions, StringSamenessResult, StringTransformFunc } from "./atomic.js"; +import { strDefaultTransforms, transforms } from "./normalization/index.js"; +declare const defaultStrategies: import("./atomic.js").ComparisonStrategy[]; declare const stringSameness: (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; declare const createStringSameness: (defaults: StringComparisonOptions) => (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; declare const strategies: { - diceStrategy: import("./atomic.js").ComparisonStrategy; - levenStrategy: import("./atomic.js").ComparisonStrategy; - cosineStrategy: import("./atomic.js").ComparisonStrategy; + diceStrategy: import("./atomic.js").ComparisonStrategy; + levenStrategy: import("./atomic.js").ComparisonStrategy; + cosineStrategy: import("./atomic.js").ComparisonStrategy; + cosineStrategyAggressive: import("./atomic.js").ComparisonStrategy; }; -export { StringSamenessResult, StringComparisonOptions, stringSameness, createStringSameness, defaultStrategies, strategies, defaultStrCompareTransformFuncs }; +declare const defaultStrCompareTransformFuncs: StringTransformFunc[]; +export { StringSamenessResult, StringComparisonOptions, stringSameness, createStringSameness, defaultStrategies, strategies, transforms, defaultStrCompareTransformFuncs, strDefaultTransforms, ComparisonStrategyResult, StringTransformFunc }; diff --git a/dist/commonjs/index.js b/dist/commonjs/index.js index 60c7b17..bcaffef 100644 --- a/dist/commonjs/index.js +++ b/dist/commonjs/index.js @@ -1,23 +1,15 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.defaultStrCompareTransformFuncs = exports.strategies = exports.defaultStrategies = exports.createStringSameness = exports.stringSameness = void 0; +exports.strDefaultTransforms = exports.defaultStrCompareTransformFuncs = exports.transforms = exports.strategies = exports.defaultStrategies = exports.createStringSameness = exports.stringSameness = void 0; const index_js_1 = require("./matchingStrategies/index.js"); +const index_js_2 = require("./normalization/index.js"); +Object.defineProperty(exports, "strDefaultTransforms", { enumerable: true, get: function () { return index_js_2.strDefaultTransforms; } }); +Object.defineProperty(exports, "transforms", { enumerable: true, get: function () { return index_js_2.transforms; } }); const sentenceLengthWeight = (length) => { // thanks jordan :') // constants are black magic return (Math.log(length) / 0.20) - 5; }; -const defaultStrCompareTransformFuncs = [ - // lower case to remove case sensitivity - (str) => str.toLocaleLowerCase(), - // remove excess whitespace - (str) => str.trim(), - // remove non-alphanumeric characters so that differences in punctuation don't subtract from comparison score - (str) => str.replace(/[^A-Za-z0-9 ]/g, ""), - // replace all instances of 2 or more whitespace with one whitespace - (str) => str.replace(/\s{2,}|\n/g, " ") -]; -exports.defaultStrCompareTransformFuncs = defaultStrCompareTransformFuncs; const defaultStrategies = [ index_js_1.diceStrategy, index_js_1.levenStrategy, @@ -25,7 +17,7 @@ const defaultStrategies = [ ]; exports.defaultStrategies = defaultStrategies; const stringSameness = (valA, valB, options) => { - const { transforms = defaultStrCompareTransformFuncs, strategies = defaultStrategies, } = options || {}; + const { transforms = index_js_2.strDefaultTransforms, strategies = defaultStrategies, } = options || {}; const cleanA = transforms.reduce((acc, curr) => curr(acc), valA); const cleanB = transforms.reduce((acc, curr) => curr(acc), valB); const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; @@ -69,6 +61,11 @@ exports.createStringSameness = createStringSameness; const strategies = { diceStrategy: index_js_1.diceStrategy, levenStrategy: index_js_1.levenStrategy, - cosineStrategy: index_js_1.cosineStrategy + cosineStrategy: index_js_1.cosineStrategy, + cosineStrategyAggressive: index_js_1.cosineStrategyAggressive }; exports.strategies = strategies; +// maintaining compatibility +const defaultStrCompareTransformFuncs = index_js_2.strDefaultTransforms; +exports.defaultStrCompareTransformFuncs = defaultStrCompareTransformFuncs; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/commonjs/index.js.map b/dist/commonjs/index.js.map new file mode 100644 index 0000000..c217079 --- /dev/null +++ b/dist/commonjs/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,4DAAoH;AAQpH,uDAA0E;AAqFtE,qGArFI,+BAAoB,OAqFJ;AAFpB,2FAnF0B,qBAAU,OAmF1B;AAjFd,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,uBAAY;IACZ,wBAAa;IACb,yBAAc;CACjB,CAAA;AAqEG,8CAAiB;AAnErB,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,+BAAoB,EACjC,UAAU,GAAG,iBAAiB,GACjC,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAEjE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AAmBG,wCAAc;AAjBlB,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAgBG,oDAAoB;AAdxB,MAAM,UAAU,GAAG;IACf,YAAY,EAAZ,uBAAY;IACZ,aAAa,EAAb,wBAAa;IACb,cAAc,EAAd,yBAAc;IACd,wBAAwB,EAAxB,mCAAwB;CAC3B,CAAC;AAWE,gCAAU;AATd,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,+BAAoB,CAAC;AAUzD,0EAA+B"} \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/cosineSimilarity.d.ts b/dist/commonjs/matchingStrategies/cosineSimilarity.d.ts index faeb8d6..1a6679e 100644 --- a/dist/commonjs/matchingStrategies/cosineSimilarity.d.ts +++ b/dist/commonjs/matchingStrategies/cosineSimilarity.d.ts @@ -1,3 +1,14 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateCosineSimilarity: (strA: string, strB: string) => number; -export declare const cosineStrategy: ComparisonStrategy; +/** + * Compares whole tokens (words) within a string independent of order + * + * This strategy is automatically disabled for strings with less than 4 words because it can lead to inaccurate scores due to not comparing characters IE it is not very useful for short sentences and comparing single words with typos + * + * If you'd like to use it even in these scenarios build your own strategy array using cosineStrategyAggressive instead of this one + * */ +export declare const cosineStrategy: ComparisonStrategy; +/** + * Always runs (strings are always valid) which may lead to inaccurate scores in low token-count strings + * */ +export declare const cosineStrategyAggressive: ComparisonStrategy; diff --git a/dist/commonjs/matchingStrategies/cosineSimilarity.js b/dist/commonjs/matchingStrategies/cosineSimilarity.js index f7b84f0..2895abe 100644 --- a/dist/commonjs/matchingStrategies/cosineSimilarity.js +++ b/dist/commonjs/matchingStrategies/cosineSimilarity.js @@ -2,7 +2,7 @@ // reproduced from https://github.com/sumn2u/string-comparison/blob/master/jscosine.js // https://sumn2u.medium.com/string-similarity-comparision-in-js-with-examples-4bae35f13968 Object.defineProperty(exports, "__esModule", { value: true }); -exports.cosineStrategy = exports.calculateCosineSimilarity = void 0; +exports.cosineStrategyAggressive = exports.cosineStrategy = exports.calculateCosineSimilarity = void 0; function termFreqMap(str) { var words = str.split(' '); var termFreq = {}; @@ -51,13 +51,41 @@ const calculateCosineSimilarity = (strA, strB) => { return cosineSimilarity(termFreqVecA, termFreqVecB); }; exports.calculateCosineSimilarity = calculateCosineSimilarity; -exports.cosineStrategy = { +const cosineBaseStrategy = { name: 'cosine', strategy: (valA, valB) => { - const res = (0, exports.calculateCosineSimilarity)(valA, valB); + let res = (0, exports.calculateCosineSimilarity)(valA, valB); + if (res > 0.99999) { + res = 1; + } return { score: res * 100, rawScore: res }; } }; +/** + * Compares whole tokens (words) within a string independent of order + * + * This strategy is automatically disabled for strings with less than 4 words because it can lead to inaccurate scores due to not comparing characters IE it is not very useful for short sentences and comparing single words with typos + * + * If you'd like to use it even in these scenarios build your own strategy array using cosineStrategyAggressive instead of this one + * */ +exports.cosineStrategy = { + ...cosineBaseStrategy, + isValid: (valA, valB) => { + // cosine only compares full tokens (words), rather than characters, in a string + // which makes its score very inaccurate when comparing low token-count strings (short sentences and/or words with typos) + // so disable its usage if there are less than 4 tokens + const valATokenLength = valA.split(' ').length; + const valBTokenLength = valB.split(' ').length; + return valATokenLength < 4 || valBTokenLength < 4; + } +}; +/** + * Always runs (strings are always valid) which may lead to inaccurate scores in low token-count strings + * */ +exports.cosineStrategyAggressive = { + ...cosineBaseStrategy +}; +//# sourceMappingURL=cosineSimilarity.js.map \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/cosineSimilarity.js.map b/dist/commonjs/matchingStrategies/cosineSimilarity.js.map new file mode 100644 index 0000000..3158aaa --- /dev/null +++ b/dist/commonjs/matchingStrategies/cosineSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"cosineSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/cosineSimilarity.ts"],"names":[],"mappings":";AAAA,sFAAsF;AACtF,2FAA2F;;;AAa3F,SAAS,WAAW,CAAC,GAAW;IAC5B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,QAAQ,GAAW,EAAE,CAAC;IAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;QACrB,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,IAAa;IAC7C,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE;QACjB,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;KACpB;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,IAAY;IAClD,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,KAAK,IAAI,IAAI,IAAI,IAAI,EAAE;QACnB,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;KACvC;IACD,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,aAAa,CAAC,IAAc,EAAE,IAAc;IACjD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAClC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KAChC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,GAAa;IAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACjC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KAC1B;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAc;IACpD,OAAO,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AACjF,CAAC;AAEM,MAAM,yBAAyB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IACpE,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC/B,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE/B,IAAI,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACxD,IAAI,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAExD,OAAO,gBAAgB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;AACxD,CAAC,CAAA;AAZY,QAAA,yBAAyB,6BAYrC;AAED,MAAM,kBAAkB,GAAuD;IAC3E,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACrC,IAAI,GAAG,GAAG,IAAA,iCAAyB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,IAAG,GAAG,GAAG,OAAO,EAAE;YACd,GAAG,GAAG,CAAC,CAAC;SACX;QACD,OAAO;YACH,KAAK,EAAE,GAAG,GAAG,GAAG;YAChB,QAAQ,EAAE,GAAG;SAChB,CAAA;IACL,CAAC;CACJ,CAAA;AAED;;;;;;KAMK;AACQ,QAAA,cAAc,GAAuD;IAC9E,GAAG,kBAAkB;IACrB,OAAO,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACpC,gFAAgF;QAChF,yHAAyH;QACzH,uDAAuD;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC/C,OAAO,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,CAAC,CAAC;IACtD,CAAC;CACJ,CAAA;AAED;;KAEK;AACQ,QAAA,wBAAwB,GAAuD;IACxF,GAAG,kBAAkB;CACxB,CAAA"} \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/diceSimilarity.d.ts b/dist/commonjs/matchingStrategies/diceSimilarity.d.ts index 6cb2a48..65d0389 100644 --- a/dist/commonjs/matchingStrategies/diceSimilarity.d.ts +++ b/dist/commonjs/matchingStrategies/diceSimilarity.d.ts @@ -1,3 +1,3 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateDiceSimilarity: (valA: string, valB: string) => number; -export declare const diceStrategy: ComparisonStrategy; +export declare const diceStrategy: ComparisonStrategy; diff --git a/dist/commonjs/matchingStrategies/diceSimilarity.js b/dist/commonjs/matchingStrategies/diceSimilarity.js index de9caf5..a83d16f 100644 --- a/dist/commonjs/matchingStrategies/diceSimilarity.js +++ b/dist/commonjs/matchingStrategies/diceSimilarity.js @@ -17,3 +17,4 @@ exports.diceStrategy = { }; } }; +//# sourceMappingURL=diceSimilarity.js.map \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/diceSimilarity.js.map b/dist/commonjs/matchingStrategies/diceSimilarity.js.map new file mode 100644 index 0000000..c06824b --- /dev/null +++ b/dist/commonjs/matchingStrategies/diceSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"diceSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/diceSimilarity.ts"],"names":[],"mappings":";;;;;;AAAA,0EAAiD;AAG1C,MAAM,uBAAuB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAAC,2BAAgB,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAAzG,QAAA,uBAAuB,2BAAkF;AACzG,QAAA,YAAY,GAAuD;IAC5E,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,IAAA,+BAAuB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,OAAO;YACH,KAAK,EAAE,GAAG,GAAG,GAAG;YAChB,QAAQ,EAAE,GAAG;SAChB,CAAA;IACL,CAAC;CACJ,CAAA"} \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/index.d.ts b/dist/commonjs/matchingStrategies/index.d.ts index a3d62ec..8c9b96f 100644 --- a/dist/commonjs/matchingStrategies/index.d.ts +++ b/dist/commonjs/matchingStrategies/index.d.ts @@ -1,5 +1,5 @@ import { levenStrategy } from "./levenSimilarity.js"; -import { cosineStrategy } from "./cosineSimilarity.js"; +import { cosineStrategy, cosineStrategyAggressive } from "./cosineSimilarity.js"; import { diceStrategy } from "./diceSimilarity.js"; import { ComparisonStrategy, ComparisonStrategyResultValue, ComparisonStrategyResultObject, StrategyFunc } from '../atomic.js'; -export { levenStrategy, cosineStrategy, diceStrategy, ComparisonStrategy, ComparisonStrategyResultObject, ComparisonStrategyResultValue, StrategyFunc }; +export { levenStrategy, cosineStrategy, cosineStrategyAggressive, diceStrategy, ComparisonStrategy, ComparisonStrategyResultObject, ComparisonStrategyResultValue, StrategyFunc }; diff --git a/dist/commonjs/matchingStrategies/index.js b/dist/commonjs/matchingStrategies/index.js index 7fdf920..d1ce17c 100644 --- a/dist/commonjs/matchingStrategies/index.js +++ b/dist/commonjs/matchingStrategies/index.js @@ -1,9 +1,11 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.diceStrategy = exports.cosineStrategy = exports.levenStrategy = void 0; +exports.diceStrategy = exports.cosineStrategyAggressive = exports.cosineStrategy = exports.levenStrategy = void 0; const levenSimilarity_js_1 = require("./levenSimilarity.js"); Object.defineProperty(exports, "levenStrategy", { enumerable: true, get: function () { return levenSimilarity_js_1.levenStrategy; } }); const cosineSimilarity_js_1 = require("./cosineSimilarity.js"); Object.defineProperty(exports, "cosineStrategy", { enumerable: true, get: function () { return cosineSimilarity_js_1.cosineStrategy; } }); +Object.defineProperty(exports, "cosineStrategyAggressive", { enumerable: true, get: function () { return cosineSimilarity_js_1.cosineStrategyAggressive; } }); const diceSimilarity_js_1 = require("./diceSimilarity.js"); Object.defineProperty(exports, "diceStrategy", { enumerable: true, get: function () { return diceSimilarity_js_1.diceStrategy; } }); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/index.js.map b/dist/commonjs/matchingStrategies/index.js.map new file mode 100644 index 0000000..a655b05 --- /dev/null +++ b/dist/commonjs/matchingStrategies/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/matchingStrategies/index.ts"],"names":[],"mappings":";;;AAAA,6DAAmD;AAM/C,8FANI,kCAAa,OAMJ;AALjB,+DAA+E;AAM3E,+FANI,oCAAc,OAMJ;AACd,yGAPoB,8CAAwB,OAOpB;AAN5B,2DAAiD;AAO7C,6FAPI,gCAAY,OAOJ"} \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/levenSimilarity.d.ts b/dist/commonjs/matchingStrategies/levenSimilarity.d.ts index 113d82d..4dee4fe 100644 --- a/dist/commonjs/matchingStrategies/levenSimilarity.d.ts +++ b/dist/commonjs/matchingStrategies/levenSimilarity.d.ts @@ -1,3 +1,3 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateLevenSimilarity: (valA: string, valB: string) => number[]; -export declare const levenStrategy: ComparisonStrategy; +export declare const levenStrategy: ComparisonStrategy; diff --git a/dist/commonjs/matchingStrategies/levenSimilarity.js b/dist/commonjs/matchingStrategies/levenSimilarity.js index d8ba8eb..ffb2786 100644 --- a/dist/commonjs/matchingStrategies/levenSimilarity.js +++ b/dist/commonjs/matchingStrategies/levenSimilarity.js @@ -17,7 +17,11 @@ const calculateLevenSimilarity = (valA, valB) => { shorter = valA; } const distance = (0, leven_1.default)(longer, shorter); - const diff = (distance / longer.length) * 100; + // use the shorter length + // because if we have more move/change operations than the length of the shortest string than we have two unique strings + // and if we use the longer length the score actually gets *better* as the length gap gets larger (not what we want) + // -- cap at 100% diff + const diff = Math.min((distance / shorter.length) * 100, 100); return [distance, 100 - diff]; }; exports.calculateLevenSimilarity = calculateLevenSimilarity; @@ -33,3 +37,4 @@ exports.levenStrategy = { name: 'leven', strategy: stratFunc }; +//# sourceMappingURL=levenSimilarity.js.map \ No newline at end of file diff --git a/dist/commonjs/matchingStrategies/levenSimilarity.js.map b/dist/commonjs/matchingStrategies/levenSimilarity.js.map new file mode 100644 index 0000000..c1ee3c1 --- /dev/null +++ b/dist/commonjs/matchingStrategies/levenSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"levenSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/levenSimilarity.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAGnB,MAAM,wBAAwB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IACnE,IAAI,MAAc,CAAC;IACnB,IAAI,OAAe,CAAC;IACpB,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE;QAC3B,MAAM,GAAG,IAAI,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;KAClB;SAAM;QACH,MAAM,GAAG,IAAI,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;KAClB;IAED,MAAM,QAAQ,GAAG,IAAA,eAAK,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,yBAAyB;IACzB,wHAAwH;IACxH,oHAAoH;IACpH,sBAAsB;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9D,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;AAClC,CAAC,CAAA;AAlBY,QAAA,wBAAwB,4BAkBpC;AAED,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAG,IAAA,gCAAwB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjD,OAAO;QACH,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QAChB,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QAChB,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;KAChB,CAAC;AACN,CAAC,CAAA;AAEY,QAAA,aAAa,GAAuD;IAC7E,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,SAAS;CACtB,CAAA"} \ No newline at end of file diff --git a/dist/commonjs/normalization/index.d.ts b/dist/commonjs/normalization/index.d.ts new file mode 100644 index 0000000..ae891c2 --- /dev/null +++ b/dist/commonjs/normalization/index.d.ts @@ -0,0 +1,17 @@ +import { StringTransformFunc } from "../atomic.js"; +declare const lowercase: StringTransformFunc; +declare const trim: StringTransformFunc; +declare const replaceUnicode: StringTransformFunc; +declare const removePunctuation: StringTransformFunc; +declare const removeWhitespace: StringTransformFunc; +declare const replaceMultiWhitespace: StringTransformFunc; +declare const transforms: { + lowercase: StringTransformFunc; + trim: StringTransformFunc; + replaceMultiWhitespace: StringTransformFunc; + replaceUnicode: StringTransformFunc; + removeWhitespace: StringTransformFunc; + removePunctuation: StringTransformFunc; +}; +declare const strDefaultTransforms: StringTransformFunc[]; +export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; diff --git a/dist/commonjs/normalization/index.js b/dist/commonjs/normalization/index.js new file mode 100644 index 0000000..73080ed --- /dev/null +++ b/dist/commonjs/normalization/index.js @@ -0,0 +1,36 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.strDefaultTransforms = exports.transforms = exports.replaceMultiWhitespace = exports.removeWhitespace = exports.removePunctuation = exports.replaceUnicode = exports.trim = exports.lowercase = void 0; +const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g); +const WHITESPACE_REGEX = new RegExp(/\s/g); +const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}|\n/g); +const lowercase = (str) => str.toLocaleLowerCase(); +exports.lowercase = lowercase; +const trim = (str) => str.trim(); +exports.trim = trim; +const replaceUnicode = (str) => str.normalize('NFD').replace(/[\u0300-\u036f]/g, ""); +exports.replaceUnicode = replaceUnicode; +const removePunctuation = (str) => str.replace(PUNCTUATION_REGEX, ''); +exports.removePunctuation = removePunctuation; +const removeWhitespace = (str) => str.replace(WHITESPACE_REGEX, ''); +exports.removeWhitespace = removeWhitespace; +const replaceMultiWhitespace = (str) => str.replace(MULTI_WHITESPACE_REGEX, ''); +exports.replaceMultiWhitespace = replaceMultiWhitespace; +const transforms = { + lowercase, + trim, + replaceMultiWhitespace, + replaceUnicode, + removeWhitespace, + removePunctuation +}; +exports.transforms = transforms; +const strDefaultTransforms = [ + replaceUnicode, + removePunctuation, + trim, + replaceMultiWhitespace, + lowercase +]; +exports.strDefaultTransforms = strDefaultTransforms; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/commonjs/normalization/index.js.map b/dist/commonjs/normalization/index.js.map new file mode 100644 index 0000000..84f643e --- /dev/null +++ b/dist/commonjs/normalization/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":";;;AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACnD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AAExD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AAyB5E,8BAAS;AAxBb,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAyB1D,oBAAI;AAxBR,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AAyB9G,wCAAc;AAxBlB,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAyB/F,8CAAiB;AAxBrB,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AAyB7F,4CAAgB;AAxBpB,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;AAyBzG,wDAAsB;AAvB1B,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAiBG,gCAAU;AAfd,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAUE,oDAAoB"} \ No newline at end of file diff --git a/dist/esm/atomic.d.ts b/dist/esm/atomic.d.ts index 6807242..deb10e9 100644 --- a/dist/esm/atomic.d.ts +++ b/dist/esm/atomic.d.ts @@ -1,6 +1,6 @@ export interface StringComparisonOptions { - transforms?: ((str: string) => string)[]; - strategies?: ComparisonStrategy[]; + transforms?: StringTransformFunc[]; + strategies?: ComparisonStrategy[]; } export interface StringSamenessResult { strategies: { @@ -26,8 +26,8 @@ export interface NamedComparisonStrategyObjectResult extends ComparisonStrategyR name: string; } export type ComparisonStrategyResultValue = number | ComparisonStrategyResultObject; -export type StrategyFunc = (strA: string, strB: string) => ComparisonStrategyResultValue; -export interface ComparisonStrategy { +export type StrategyFunc = (strA: string, strB: string) => T; +export interface ComparisonStrategy { /** * The name of this strategy * */ @@ -35,9 +35,10 @@ export interface ComparisonStrategy { /** * A function that accepts two string arguments and returns a number * */ - strategy: StrategyFunc; + strategy: StrategyFunc; /** * An optional function that accepts two string arguments and returns whether this strategy should be used * */ isValid?: (strA: string, strB: string) => boolean; } +export type StringTransformFunc = (str: string) => string; diff --git a/dist/esm/atomic.js b/dist/esm/atomic.js index cb0ff5c..7730b0a 100644 --- a/dist/esm/atomic.js +++ b/dist/esm/atomic.js @@ -1 +1,2 @@ export {}; +//# sourceMappingURL=atomic.js.map \ No newline at end of file diff --git a/dist/esm/atomic.js.map b/dist/esm/atomic.js.map new file mode 100644 index 0000000..d76053d --- /dev/null +++ b/dist/esm/atomic.js.map @@ -0,0 +1 @@ +{"version":3,"file":"atomic.js","sourceRoot":"","sources":["../../src/atomic.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/dist/esm/index.d.ts b/dist/esm/index.d.ts index 8c1309f..a9ac7c2 100644 --- a/dist/esm/index.d.ts +++ b/dist/esm/index.d.ts @@ -1,11 +1,13 @@ -import { StringComparisonOptions, StringSamenessResult } from "./atomic.js"; -declare const defaultStrCompareTransformFuncs: ((str: string) => string)[]; -declare const defaultStrategies: import("./atomic.js").ComparisonStrategy[]; +import { ComparisonStrategyResult, StringComparisonOptions, StringSamenessResult, StringTransformFunc } from "./atomic.js"; +import { strDefaultTransforms, transforms } from "./normalization/index.js"; +declare const defaultStrategies: import("./atomic.js").ComparisonStrategy[]; declare const stringSameness: (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; declare const createStringSameness: (defaults: StringComparisonOptions) => (valA: string, valB: string, options?: StringComparisonOptions) => StringSamenessResult; declare const strategies: { - diceStrategy: import("./atomic.js").ComparisonStrategy; - levenStrategy: import("./atomic.js").ComparisonStrategy; - cosineStrategy: import("./atomic.js").ComparisonStrategy; + diceStrategy: import("./atomic.js").ComparisonStrategy; + levenStrategy: import("./atomic.js").ComparisonStrategy; + cosineStrategy: import("./atomic.js").ComparisonStrategy; + cosineStrategyAggressive: import("./atomic.js").ComparisonStrategy; }; -export { StringSamenessResult, StringComparisonOptions, stringSameness, createStringSameness, defaultStrategies, strategies, defaultStrCompareTransformFuncs }; +declare const defaultStrCompareTransformFuncs: StringTransformFunc[]; +export { StringSamenessResult, StringComparisonOptions, stringSameness, createStringSameness, defaultStrategies, strategies, transforms, defaultStrCompareTransformFuncs, strDefaultTransforms, ComparisonStrategyResult, StringTransformFunc }; diff --git a/dist/esm/index.js b/dist/esm/index.js index 9cb183b..9740d73 100644 --- a/dist/esm/index.js +++ b/dist/esm/index.js @@ -1,26 +1,17 @@ -import { cosineStrategy, levenStrategy, diceStrategy } from "./matchingStrategies/index.js"; +import { cosineStrategy, levenStrategy, diceStrategy, cosineStrategyAggressive } from "./matchingStrategies/index.js"; +import { strDefaultTransforms, transforms } from "./normalization/index.js"; const sentenceLengthWeight = (length) => { // thanks jordan :') // constants are black magic return (Math.log(length) / 0.20) - 5; }; -const defaultStrCompareTransformFuncs = [ - // lower case to remove case sensitivity - (str) => str.toLocaleLowerCase(), - // remove excess whitespace - (str) => str.trim(), - // remove non-alphanumeric characters so that differences in punctuation don't subtract from comparison score - (str) => str.replace(/[^A-Za-z0-9 ]/g, ""), - // replace all instances of 2 or more whitespace with one whitespace - (str) => str.replace(/\s{2,}|\n/g, " ") -]; const defaultStrategies = [ diceStrategy, levenStrategy, cosineStrategy ]; const stringSameness = (valA, valB, options) => { - const { transforms = defaultStrCompareTransformFuncs, strategies = defaultStrategies, } = options || {}; + const { transforms = strDefaultTransforms, strategies = defaultStrategies, } = options || {}; const cleanA = transforms.reduce((acc, curr) => curr(acc), valA); const cleanB = transforms.reduce((acc, curr) => curr(acc), valB); const shortest = cleanA.length > cleanB.length ? cleanB : cleanA; @@ -62,6 +53,10 @@ const createStringSameness = (defaults) => { const strategies = { diceStrategy, levenStrategy, - cosineStrategy + cosineStrategy, + cosineStrategyAggressive }; -export { stringSameness, createStringSameness, defaultStrategies, strategies, defaultStrCompareTransformFuncs }; +// maintaining compatibility +const defaultStrCompareTransformFuncs = strDefaultTransforms; +export { stringSameness, createStringSameness, defaultStrategies, strategies, transforms, defaultStrCompareTransformFuncs, strDefaultTransforms }; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/esm/index.js.map b/dist/esm/index.js.map new file mode 100644 index 0000000..c57cb68 --- /dev/null +++ b/dist/esm/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,wBAAwB,EAAC,MAAM,+BAA+B,CAAC;AAQpH,OAAO,EAAC,oBAAoB,EAAE,UAAU,EAAC,MAAM,0BAA0B,CAAC;AAE1E,MAAM,oBAAoB,GAAG,CAAC,MAAc,EAAE,EAAE;IAC5C,oBAAoB;IACpB,4BAA4B;IAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC,CAAA;AAED,MAAM,iBAAiB,GAAG;IACtB,YAAY;IACZ,aAAa;IACb,cAAc;CACjB,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,OAAiC,EAAwB,EAAE;IAE3G,MAAM,EACF,UAAU,GAAG,oBAAoB,EACjC,UAAU,GAAG,iBAAiB,GACjC,GAAG,OAAO,IAAI,EAAE,CAAC;IAElB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAEjE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,YAAY,GAA0C,EAAE,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;YAC/D,SAAS;SACZ;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAC,KAAK,EAAE,GAAG,EAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,YAAY,CAAC,IAAI,CAAC;YACd,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;KACN;IAED,mCAAmC;IACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE1D,qBAAqB;IACrB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC;IAChG,kCAAkC;IAClC,MAAM,iBAAiB,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAgD,EAAE,IAAI,EAAE,EAAE;QAC5F,MAAM,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAC,GAAG,IAAI,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACb,GAAG,IAAI;YACP,KAAK;SACR,CAAC;QACF,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,OAAO;QACH,UAAU,EAAE,QAAQ;QACpB,SAAS;QACT,iBAAiB;KACpB,CAAA;AACL,CAAC,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,QAAiC,EAAE,EAAE;IAC/D,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,UAAmC,EAAE,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,EAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAC,CAAC,CAAC;AACxI,CAAC,CAAA;AAED,MAAM,UAAU,GAAG;IACf,YAAY;IACZ,aAAa;IACb,cAAc;IACd,wBAAwB;CAC3B,CAAC;AAEF,4BAA4B;AAC5B,MAAM,+BAA+B,GAAG,oBAAoB,CAAC;AAE7D,OAAO,EAGH,cAAc,EACd,oBAAoB,EACpB,iBAAiB,EACjB,UAAU,EACV,UAAU,EACV,+BAA+B,EAC/B,oBAAoB,EAGvB,CAAA"} \ No newline at end of file diff --git a/dist/esm/matchingStrategies/cosineSimilarity.d.ts b/dist/esm/matchingStrategies/cosineSimilarity.d.ts index faeb8d6..1a6679e 100644 --- a/dist/esm/matchingStrategies/cosineSimilarity.d.ts +++ b/dist/esm/matchingStrategies/cosineSimilarity.d.ts @@ -1,3 +1,14 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateCosineSimilarity: (strA: string, strB: string) => number; -export declare const cosineStrategy: ComparisonStrategy; +/** + * Compares whole tokens (words) within a string independent of order + * + * This strategy is automatically disabled for strings with less than 4 words because it can lead to inaccurate scores due to not comparing characters IE it is not very useful for short sentences and comparing single words with typos + * + * If you'd like to use it even in these scenarios build your own strategy array using cosineStrategyAggressive instead of this one + * */ +export declare const cosineStrategy: ComparisonStrategy; +/** + * Always runs (strings are always valid) which may lead to inaccurate scores in low token-count strings + * */ +export declare const cosineStrategyAggressive: ComparisonStrategy; diff --git a/dist/esm/matchingStrategies/cosineSimilarity.js b/dist/esm/matchingStrategies/cosineSimilarity.js index 318a709..f0623b3 100644 --- a/dist/esm/matchingStrategies/cosineSimilarity.js +++ b/dist/esm/matchingStrategies/cosineSimilarity.js @@ -47,13 +47,41 @@ export const calculateCosineSimilarity = (strA, strB) => { var termFreqVecB = termFreqMapToVector(termFreqB, dict); return cosineSimilarity(termFreqVecA, termFreqVecB); }; -export const cosineStrategy = { +const cosineBaseStrategy = { name: 'cosine', strategy: (valA, valB) => { - const res = calculateCosineSimilarity(valA, valB); + let res = calculateCosineSimilarity(valA, valB); + if (res > 0.99999) { + res = 1; + } return { score: res * 100, rawScore: res }; } }; +/** + * Compares whole tokens (words) within a string independent of order + * + * This strategy is automatically disabled for strings with less than 4 words because it can lead to inaccurate scores due to not comparing characters IE it is not very useful for short sentences and comparing single words with typos + * + * If you'd like to use it even in these scenarios build your own strategy array using cosineStrategyAggressive instead of this one + * */ +export const cosineStrategy = { + ...cosineBaseStrategy, + isValid: (valA, valB) => { + // cosine only compares full tokens (words), rather than characters, in a string + // which makes its score very inaccurate when comparing low token-count strings (short sentences and/or words with typos) + // so disable its usage if there are less than 4 tokens + const valATokenLength = valA.split(' ').length; + const valBTokenLength = valB.split(' ').length; + return valATokenLength < 4 || valBTokenLength < 4; + } +}; +/** + * Always runs (strings are always valid) which may lead to inaccurate scores in low token-count strings + * */ +export const cosineStrategyAggressive = { + ...cosineBaseStrategy +}; +//# sourceMappingURL=cosineSimilarity.js.map \ No newline at end of file diff --git a/dist/esm/matchingStrategies/cosineSimilarity.js.map b/dist/esm/matchingStrategies/cosineSimilarity.js.map new file mode 100644 index 0000000..eaef648 --- /dev/null +++ b/dist/esm/matchingStrategies/cosineSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"cosineSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/cosineSimilarity.ts"],"names":[],"mappings":"AAAA,sFAAsF;AACtF,2FAA2F;AAa3F,SAAS,WAAW,CAAC,GAAW;IAC5B,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,QAAQ,GAAW,EAAE,CAAC;IAC1B,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;QACrB,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,IAAa;IAC7C,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE;QACjB,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;KACpB;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,IAAY;IAClD,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,KAAK,IAAI,IAAI,IAAI,IAAI,EAAE;QACnB,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;KACvC;IACD,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,aAAa,CAAC,IAAc,EAAE,IAAc;IACjD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAClC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KAChC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,GAAa;IAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACjC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KAC1B;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAc;IACpD,OAAO,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IACpE,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAElC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC/B,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE/B,IAAI,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACxD,IAAI,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAExD,OAAO,gBAAgB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;AACxD,CAAC,CAAA;AAED,MAAM,kBAAkB,GAAuD;IAC3E,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACrC,IAAI,GAAG,GAAG,yBAAyB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,IAAG,GAAG,GAAG,OAAO,EAAE;YACd,GAAG,GAAG,CAAC,CAAC;SACX;QACD,OAAO;YACH,KAAK,EAAE,GAAG,GAAG,GAAG;YAChB,QAAQ,EAAE,GAAG;SAChB,CAAA;IACL,CAAC;CACJ,CAAA;AAED;;;;;;KAMK;AACL,MAAM,CAAC,MAAM,cAAc,GAAuD;IAC9E,GAAG,kBAAkB;IACrB,OAAO,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACpC,gFAAgF;QAChF,yHAAyH;QACzH,uDAAuD;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QAC/C,OAAO,eAAe,GAAG,CAAC,IAAI,eAAe,GAAG,CAAC,CAAC;IACtD,CAAC;CACJ,CAAA;AAED;;KAEK;AACL,MAAM,CAAC,MAAM,wBAAwB,GAAuD;IACxF,GAAG,kBAAkB;CACxB,CAAA"} \ No newline at end of file diff --git a/dist/esm/matchingStrategies/diceSimilarity.d.ts b/dist/esm/matchingStrategies/diceSimilarity.d.ts index 6cb2a48..65d0389 100644 --- a/dist/esm/matchingStrategies/diceSimilarity.d.ts +++ b/dist/esm/matchingStrategies/diceSimilarity.d.ts @@ -1,3 +1,3 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateDiceSimilarity: (valA: string, valB: string) => number; -export declare const diceStrategy: ComparisonStrategy; +export declare const diceStrategy: ComparisonStrategy; diff --git a/dist/esm/matchingStrategies/diceSimilarity.js b/dist/esm/matchingStrategies/diceSimilarity.js index 595377a..cd00b1a 100644 --- a/dist/esm/matchingStrategies/diceSimilarity.js +++ b/dist/esm/matchingStrategies/diceSimilarity.js @@ -10,3 +10,4 @@ export const diceStrategy = { }; } }; +//# sourceMappingURL=diceSimilarity.js.map \ No newline at end of file diff --git a/dist/esm/matchingStrategies/diceSimilarity.js.map b/dist/esm/matchingStrategies/diceSimilarity.js.map new file mode 100644 index 0000000..cb4e84e --- /dev/null +++ b/dist/esm/matchingStrategies/diceSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"diceSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/diceSimilarity.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,mBAAmB,CAAC;AAGjD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACtH,MAAM,CAAC,MAAM,YAAY,GAAuD;IAC5E,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,uBAAuB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,OAAO;YACH,KAAK,EAAE,GAAG,GAAG,GAAG;YAChB,QAAQ,EAAE,GAAG;SAChB,CAAA;IACL,CAAC;CACJ,CAAA"} \ No newline at end of file diff --git a/dist/esm/matchingStrategies/index.d.ts b/dist/esm/matchingStrategies/index.d.ts index a3d62ec..8c9b96f 100644 --- a/dist/esm/matchingStrategies/index.d.ts +++ b/dist/esm/matchingStrategies/index.d.ts @@ -1,5 +1,5 @@ import { levenStrategy } from "./levenSimilarity.js"; -import { cosineStrategy } from "./cosineSimilarity.js"; +import { cosineStrategy, cosineStrategyAggressive } from "./cosineSimilarity.js"; import { diceStrategy } from "./diceSimilarity.js"; import { ComparisonStrategy, ComparisonStrategyResultValue, ComparisonStrategyResultObject, StrategyFunc } from '../atomic.js'; -export { levenStrategy, cosineStrategy, diceStrategy, ComparisonStrategy, ComparisonStrategyResultObject, ComparisonStrategyResultValue, StrategyFunc }; +export { levenStrategy, cosineStrategy, cosineStrategyAggressive, diceStrategy, ComparisonStrategy, ComparisonStrategyResultObject, ComparisonStrategyResultValue, StrategyFunc }; diff --git a/dist/esm/matchingStrategies/index.js b/dist/esm/matchingStrategies/index.js index f55d695..2cdbf8c 100644 --- a/dist/esm/matchingStrategies/index.js +++ b/dist/esm/matchingStrategies/index.js @@ -1,4 +1,5 @@ import { levenStrategy } from "./levenSimilarity.js"; -import { cosineStrategy } from "./cosineSimilarity.js"; +import { cosineStrategy, cosineStrategyAggressive } from "./cosineSimilarity.js"; import { diceStrategy } from "./diceSimilarity.js"; -export { levenStrategy, cosineStrategy, diceStrategy }; +export { levenStrategy, cosineStrategy, cosineStrategyAggressive, diceStrategy }; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/esm/matchingStrategies/index.js.map b/dist/esm/matchingStrategies/index.js.map new file mode 100644 index 0000000..5e4098b --- /dev/null +++ b/dist/esm/matchingStrategies/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/matchingStrategies/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAC,cAAc,EAAE,wBAAwB,EAAC,MAAM,uBAAuB,CAAC;AAC/E,OAAO,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAGjD,OAAO,EACH,aAAa,EACb,cAAc,EACd,wBAAwB,EACxB,YAAY,EAKf,CAAA"} \ No newline at end of file diff --git a/dist/esm/matchingStrategies/levenSimilarity.d.ts b/dist/esm/matchingStrategies/levenSimilarity.d.ts index 113d82d..4dee4fe 100644 --- a/dist/esm/matchingStrategies/levenSimilarity.d.ts +++ b/dist/esm/matchingStrategies/levenSimilarity.d.ts @@ -1,3 +1,3 @@ -import { ComparisonStrategy } from "../atomic.js"; +import { ComparisonStrategy, ComparisonStrategyResultObject } from "../atomic.js"; export declare const calculateLevenSimilarity: (valA: string, valB: string) => number[]; -export declare const levenStrategy: ComparisonStrategy; +export declare const levenStrategy: ComparisonStrategy; diff --git a/dist/esm/matchingStrategies/levenSimilarity.js b/dist/esm/matchingStrategies/levenSimilarity.js index 0e4f765..3852db8 100644 --- a/dist/esm/matchingStrategies/levenSimilarity.js +++ b/dist/esm/matchingStrategies/levenSimilarity.js @@ -11,7 +11,11 @@ export const calculateLevenSimilarity = (valA, valB) => { shorter = valA; } const distance = leven(longer, shorter); - const diff = (distance / longer.length) * 100; + // use the shorter length + // because if we have more move/change operations than the length of the shortest string than we have two unique strings + // and if we use the longer length the score actually gets *better* as the length gap gets larger (not what we want) + // -- cap at 100% diff + const diff = Math.min((distance / shorter.length) * 100, 100); return [distance, 100 - diff]; }; const stratFunc = (valA, valB) => { @@ -26,3 +30,4 @@ export const levenStrategy = { name: 'leven', strategy: stratFunc }; +//# sourceMappingURL=levenSimilarity.js.map \ No newline at end of file diff --git a/dist/esm/matchingStrategies/levenSimilarity.js.map b/dist/esm/matchingStrategies/levenSimilarity.js.map new file mode 100644 index 0000000..a8faf41 --- /dev/null +++ b/dist/esm/matchingStrategies/levenSimilarity.js.map @@ -0,0 +1 @@ +{"version":3,"file":"levenSimilarity.js","sourceRoot":"","sources":["../../../src/matchingStrategies/levenSimilarity.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAG1B,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IACnE,IAAI,MAAc,CAAC;IACnB,IAAI,OAAe,CAAC;IACpB,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE;QAC3B,MAAM,GAAG,IAAI,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;KAClB;SAAM;QACH,MAAM,GAAG,IAAI,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;KAClB;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,yBAAyB;IACzB,wHAAwH;IACxH,oHAAoH;IACpH,sBAAsB;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAC9D,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;AAClC,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjD,OAAO;QACH,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QAChB,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QAChB,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;KAChB,CAAC;AACN,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,aAAa,GAAuD;IAC7E,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,SAAS;CACtB,CAAA"} \ No newline at end of file diff --git a/dist/esm/normalization/index.d.ts b/dist/esm/normalization/index.d.ts new file mode 100644 index 0000000..ae891c2 --- /dev/null +++ b/dist/esm/normalization/index.d.ts @@ -0,0 +1,17 @@ +import { StringTransformFunc } from "../atomic.js"; +declare const lowercase: StringTransformFunc; +declare const trim: StringTransformFunc; +declare const replaceUnicode: StringTransformFunc; +declare const removePunctuation: StringTransformFunc; +declare const removeWhitespace: StringTransformFunc; +declare const replaceMultiWhitespace: StringTransformFunc; +declare const transforms: { + lowercase: StringTransformFunc; + trim: StringTransformFunc; + replaceMultiWhitespace: StringTransformFunc; + replaceUnicode: StringTransformFunc; + removeWhitespace: StringTransformFunc; + removePunctuation: StringTransformFunc; +}; +declare const strDefaultTransforms: StringTransformFunc[]; +export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; diff --git a/dist/esm/normalization/index.js b/dist/esm/normalization/index.js new file mode 100644 index 0000000..8029f5d --- /dev/null +++ b/dist/esm/normalization/index.js @@ -0,0 +1,26 @@ +const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g); +const WHITESPACE_REGEX = new RegExp(/\s/g); +const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}|\n/g); +const lowercase = (str) => str.toLocaleLowerCase(); +const trim = (str) => str.trim(); +const replaceUnicode = (str) => str.normalize('NFD').replace(/[\u0300-\u036f]/g, ""); +const removePunctuation = (str) => str.replace(PUNCTUATION_REGEX, ''); +const removeWhitespace = (str) => str.replace(WHITESPACE_REGEX, ''); +const replaceMultiWhitespace = (str) => str.replace(MULTI_WHITESPACE_REGEX, ''); +const transforms = { + lowercase, + trim, + replaceMultiWhitespace, + replaceUnicode, + removeWhitespace, + removePunctuation +}; +const strDefaultTransforms = [ + replaceUnicode, + removePunctuation, + trim, + replaceMultiWhitespace, + lowercase +]; +export { lowercase, trim, replaceUnicode, removePunctuation, removeWhitespace, replaceMultiWhitespace, transforms, strDefaultTransforms }; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/dist/esm/normalization/index.js.map b/dist/esm/normalization/index.js.map new file mode 100644 index 0000000..68fe96f --- /dev/null +++ b/dist/esm/normalization/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/normalization/index.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AACnD,MAAM,gBAAgB,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3C,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;AAExD,MAAM,SAAS,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;AAChF,MAAM,IAAI,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAC9D,MAAM,cAAc,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AAClH,MAAM,iBAAiB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AACnG,MAAM,gBAAgB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACjG,MAAM,sBAAsB,GAAwB,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC;AAE7G,MAAM,UAAU,GAAG;IACf,SAAS;IACT,IAAI;IACJ,sBAAsB;IACtB,cAAc;IACd,gBAAgB;IAChB,iBAAiB;CACpB,CAAA;AAED,MAAM,oBAAoB,GAA0B;IAChD,cAAc;IACd,iBAAiB;IACjB,IAAI;IACJ,sBAAsB;IACtB,SAAS;CACZ,CAAC;AAEF,OAAO,EACH,SAAS,EACT,IAAI,EACJ,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,sBAAsB,EACtB,UAAU,EACV,oBAAoB,EACvB,CAAA"} \ No newline at end of file diff --git a/package.json b/package.json index ef1425a..ad5ef6a 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "typedoc": "typedoc", "build": "tsc", "prepare": "tshy", - "test": "mocha --recursive ./tests/*.test.ts" + "test": "mocha --recursive ./tests/*.test.ts" }, "exports": { "./package.json": "./package.json", diff --git a/src/atomic.ts b/src/atomic.ts index 1d73f02..37aacdf 100644 --- a/src/atomic.ts +++ b/src/atomic.ts @@ -1,5 +1,5 @@ export interface StringComparisonOptions { - transforms?: ((str: string) => string)[] + transforms?: StringTransformFunc[] strategies?: ComparisonStrategy[] } @@ -52,3 +52,5 @@ export interface ComparisonStrategy { * */ isValid?: (strA: string, strB: string) => boolean } + +export type StringTransformFunc = (str: string) => string; diff --git a/src/index.ts b/src/index.ts index 5d70858..06063c2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,8 +3,10 @@ import { ComparisonStrategyResult, NamedComparisonStrategyObjectResult, StringComparisonOptions, - StringSamenessResult + StringSamenessResult, + StringTransformFunc } from "./atomic.js"; +import {strDefaultTransforms, transforms} from "./normalization/index.js"; const sentenceLengthWeight = (length: number) => { // thanks jordan :') @@ -12,17 +14,6 @@ const sentenceLengthWeight = (length: number) => { return (Math.log(length) / 0.20) - 5; } -const defaultStrCompareTransformFuncs = [ - // lower case to remove case sensitivity - (str: string) => str.toLocaleLowerCase(), - // remove excess whitespace - (str: string) => str.trim(), - // remove non-alphanumeric characters so that differences in punctuation don't subtract from comparison score - (str: string) => str.replace(/[^A-Za-z0-9 ]/g, ""), - // replace all instances of 2 or more whitespace with one whitespace - (str: string) => str.replace(/\s{2,}|\n/g, " ") -]; - const defaultStrategies = [ diceStrategy, levenStrategy, @@ -32,7 +23,7 @@ const defaultStrategies = [ const stringSameness = (valA: string, valB: string, options?: StringComparisonOptions): StringSamenessResult => { const { - transforms = defaultStrCompareTransformFuncs, + transforms = strDefaultTransforms, strategies = defaultStrategies, } = options || {}; @@ -88,6 +79,9 @@ const strategies = { cosineStrategyAggressive }; +// maintaining compatibility +const defaultStrCompareTransformFuncs = strDefaultTransforms; + export { StringSamenessResult, StringComparisonOptions, @@ -95,6 +89,9 @@ export { createStringSameness, defaultStrategies, strategies, + transforms, defaultStrCompareTransformFuncs, - ComparisonStrategyResult + strDefaultTransforms, + ComparisonStrategyResult, + StringTransformFunc } diff --git a/src/normalization/index.ts b/src/normalization/index.ts new file mode 100644 index 0000000..61c85b7 --- /dev/null +++ b/src/normalization/index.ts @@ -0,0 +1,40 @@ +import {StringTransformFunc} from "../atomic.js"; + +const PUNCTUATION_REGEX = new RegExp(/[^\w\s]|_/g); +const WHITESPACE_REGEX = new RegExp(/\s/g); +const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g); + +const lowercase: StringTransformFunc = (str: string) => str.toLocaleLowerCase(); +const trim: StringTransformFunc = (str: string) => str.trim(); +const replaceUnicode: StringTransformFunc = (str: string) => str.normalize('NFD').replace(/[\u0300-\u036f]/g, ""); +const removePunctuation: StringTransformFunc = (str: string) => str.replace(PUNCTUATION_REGEX, ''); +const removeWhitespace: StringTransformFunc = (str: string) => str.replace(WHITESPACE_REGEX, ''); +const replaceMultiWhitespace: StringTransformFunc = (str: string) => str.replace(MULTI_WHITESPACE_REGEX, ' '); + +const transforms = { + lowercase, + trim, + replaceMultiWhitespace, + replaceUnicode, + removeWhitespace, + removePunctuation +} + +const strDefaultTransforms: StringTransformFunc[] = [ + replaceUnicode, + removePunctuation, + trim, + replaceMultiWhitespace, + lowercase +]; + +export { + lowercase, + trim, + replaceUnicode, + removePunctuation, + removeWhitespace, + replaceMultiWhitespace, + transforms, + strDefaultTransforms +} diff --git a/tests/index.test.ts b/tests/index.test.ts index 41fcc53..a82c8f1 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -16,33 +16,43 @@ describe('String normalization', function() { it('should normalize punctuation', async function () { const valA = 'this string! is the. same'; - const valB = 'this string is the same'; - assert.equal(normalizeStr(valA), normalizeStr(valB)) + assert.equal(normalizeStr(valA), sameString) }); it('should normalize whitespace', async function () { const valA = 'this string is the same '; - const valB = 'this string is the same'; - assert.equal(normalizeStr(valA), normalizeStr(valB)) + assert.equal(normalizeStr(valA), sameString) }); it('should normalize whitespace at end and beginning', async function () { const valA = ' this string is the same '; - const valB = 'this string is the same'; - assert.equal(normalizeStr(valA), normalizeStr(valB)) + assert.equal(normalizeStr(valA), sameString) }); it('should normalize case', async function () { const valA = 'ThIs STRING iS THe SAMe'; - const valB = 'this string is the same'; - assert.equal(normalizeStr(valA), normalizeStr(valB)) + assert.equal(normalizeStr(valA), sameString) + }); + + it('should normalize unicode', async function () { + + const tests = [ + ['enamorate bailando','Enamórate Bailando'], + ['dina ogon', 'Dina Ögon'], + ['nana', 'Nanã'], + ['nana', 'Nanä'] + ] + + for(const test of tests) { + assert.equal(normalizeStr(test[1]), test[0]) + } }); }); diff --git a/typedoc.json b/typedoc.json index d01e4ef..e9ff722 100644 --- a/typedoc.json +++ b/typedoc.json @@ -1,7 +1,11 @@ { "$schema": "https://typedoc.org/schema.json", "name": "string-sameness Docs", - "entryPoints": ["./src","./src/matchingStrategies/index.ts"], + "entryPoints": [ + "./src", + "./src/matchingStrategies/index.ts", + "./src/normalization/index.ts" + ], "sort": ["source-order"], "categorizeByGroup": false, "searchGroupBoosts": {