diff --git a/packages/lu/package.json b/packages/lu/package.json index 7508338c7..5e4c37e06 100644 --- a/packages/lu/package.json +++ b/packages/lu/package.json @@ -47,7 +47,7 @@ "get-stdin": "^6.0.0", "globby": "^10.0.1", "intercept-stdout": "^0.1.2", - "lodash": "^4.17.15", + "lodash": "^4.17.19", "node-fetch": "~2.6.0", "semver": "^5.5.1", "tslib": "^1.10.0" diff --git a/packages/lu/src/parser/lufile/baseSection.js b/packages/lu/src/parser/lufile/baseSection.js new file mode 100644 index 000000000..8247a67b5 --- /dev/null +++ b/packages/lu/src/parser/lufile/baseSection.js @@ -0,0 +1,19 @@ +class BaseSection { + constructor(parameters) { + this.Errors = []; + this.SectionType = ''; + this.Id = ''; + this.Body = ''; + this.Range; + + if (parameters) { + this.Errors = parameters.Errors || []; + this.SectionType = parameters.SectionType || ''; + this.Id = parameters.Id || ''; + this.Body = parameters.Body || ''; + this.Range = parameters.Range; + } + } +} + +module.exports = BaseSection; \ No newline at end of file diff --git a/packages/lu/src/parser/lufile/diagnostic.js b/packages/lu/src/parser/lufile/diagnostic.js index 6fbfe1317..8e9eeb4fa 100644 --- a/packages/lu/src/parser/lufile/diagnostic.js +++ b/packages/lu/src/parser/lufile/diagnostic.js @@ -9,7 +9,6 @@ class Diagnostic { } toString() { - // ignore error range if source is "inline" if (this.Range === undefined) { return `[${DiagnosticSeverity[this.Severity]}] ${this.Message.toString()}`; } @@ -78,8 +77,13 @@ const BuildDiagnostic =function(parameter) { const severity = parameter.severity === undefined ? DiagnosticSeverity.ERROR : parameter.severity; let range; + const rangeInput = parameter.range; const context = parameter.context; - if (context !== undefined) { + if (rangeInput !== undefined) { + const startPosition = new Position(rangeInput.Start.Line, rangeInput.Start.Character); + const stopPosition = new Position(rangeInput.End.Line, rangeInput.End.Character); + range = new Range(startPosition, stopPosition); + } else if (context !== undefined) { const startPosition = new Position(context.start.line, context.start.column); const stopPosition = new Position(context.stop.line, context.stop.column + context.stop.text.length); range = new Range(startPosition, stopPosition); diff --git a/packages/lu/src/parser/lufile/entitySection.js b/packages/lu/src/parser/lufile/entitySection.js index 0b1f01168..887a4f714 100644 --- a/packages/lu/src/parser/lufile/entitySection.js +++ b/packages/lu/src/parser/lufile/entitySection.js @@ -3,20 +3,25 @@ const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity; const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; const LUSectionTypes = require('./../utils/enums/lusectiontypes'); const InvalidCharsInIntentOrEntityName = require('./../utils/enums/invalidchars').InvalidCharsInIntentOrEntityName; +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class EntitySection { +class EntitySection extends BaseSection { /** * * @param {EntitySectionContext} parseTree */ constructor(parseTree) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.ENTITYSECTION; - this.Errors = [] this.Name = this.ExtractName(parseTree); this.Type = this.ExtractType(parseTree); this.SynonymsOrPhraseList = this.ExtractSynonymsOrPhraseList(parseTree); this.Id = `${this.SectionType}_${this.Name}`; + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } ExtractName(parseTree) { diff --git a/packages/lu/src/parser/lufile/importSection.js b/packages/lu/src/parser/lufile/importSection.js index 574a485cb..901cb8c39 100644 --- a/packages/lu/src/parser/lufile/importSection.js +++ b/packages/lu/src/parser/lufile/importSection.js @@ -1,20 +1,26 @@ const ImportSectionContext = require('./generated/LUFileParser').LUFileParser.ImportSectionContext; const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; const LUSectionTypes = require('./../utils/enums/lusectiontypes'); +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class ImportSection { +class ImportSection extends BaseSection { /** * * @param {ImportSectionContext} parseTree */ constructor(parseTree) { - this.ParseTree = parseTree; + super(); this.Errors = [] this.SectionType = LUSectionTypes.IMPORTSECTION; let result = this.ExtractDescriptionAndPath(parseTree); this.Description = result.description; this.Path = result.path; - this.Id = `${this.SectionType}_${this.Path}`;; + this.Id = `${this.SectionType}_${this.Path}`; + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } ExtractDescriptionAndPath(parseTree) { diff --git a/packages/lu/src/parser/lufile/luParser.js b/packages/lu/src/parser/lufile/luParser.js index 92cbd8ca0..1a5279088 100644 --- a/packages/lu/src/parser/lufile/luParser.js +++ b/packages/lu/src/parser/lufile/luParser.js @@ -14,22 +14,42 @@ const LUErrorListener = require('./luErrorListener'); const SectionType = require('./../utils/enums/lusectiontypes'); const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity; const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; const NEWLINE = require('os').EOL; class LUParser { + /** - * @param {string} text + * + * @param {string} text + * @param {LUResource} luResource */ - static parse(text) { + static parseWithRef(text, luResource) { if (text === undefined || text === '') { return new LUResource([], '', []); } - let sections = []; - let content = text; + const sectionEnabled = luResource ? this.isSectionEnabled(luResource.Sections) : undefined; + + return this.parse(text, sectionEnabled); + } + + /** + * @param {string} text + */ + static parse(text, sectionEnabled) { + if (text === undefined || text === '') { + return new LUResource([], '', []); + } let {fileContent, errors} = this.getFileContent(text); + return this.extractFileContent(fileContent, text, errors, sectionEnabled); + } + + static extractFileContent(fileContent, content, errors, sectionEnabled) { + let sections = []; try { let modelInfoSections = this.extractModelInfoSections(fileContent); modelInfoSections.forEach(section => errors = errors.concat(section.Errors)); @@ -41,7 +61,7 @@ class LUParser { } try { - let isSectionEnabled = this.isSectionEnabled(sections); + let isSectionEnabled = sectionEnabled === undefined ? this.isSectionEnabled(sections) : sectionEnabled; let nestedIntentSections = this.extractNestedIntentSections(fileContent, content); nestedIntentSections.forEach(section => errors = errors.concat(section.Errors)); @@ -50,12 +70,21 @@ class LUParser { } else { nestedIntentSections.forEach(section => { let emptyIntentSection = new SimpleIntentSection(); - emptyIntentSection.ParseTree = section.ParseTree.nestedIntentNameLine(); emptyIntentSection.Name = section.Name; + emptyIntentSection.Id = `${emptyIntentSection.SectionType}_${emptyIntentSection.Name}` + + // get the end character index + const firstLine = content.split(/\r?\n/)[0]; + let endCharacter = section.Name.length + 2; + if (firstLine.includes(section.Name)) { + endCharacter = firstLine.length; + } + const range = new Range(section.Range.Start, new Position(section.Range.Start.Line, endCharacter)) + emptyIntentSection.Range = range; let errorMsg = `no utterances found for intent definition: "# ${emptyIntentSection.Name}"` let error = BuildDiagnostic({ message: errorMsg, - context: emptyIntentSection.ParseTree, + range: emptyIntentSection.Range, severity: DiagnosticSeverity.WARN }) @@ -301,23 +330,24 @@ class LUParser { * @param {string} content */ static extractSectionBody(sections, content) { - sections.sort((a, b) => a.ParseTree.start.line - b.ParseTree.start.line) + sections.sort((a, b) => a.Range.Start.Line - b.Range.Start.Line) const originList = content.split(/\r?\n/) let qnaSectionIndex = 0 sections.forEach(function (section, index) { if (section.SectionType === SectionType.SIMPLEINTENTSECTION || section.SectionType === SectionType.NESTEDINTENTSECTION || section.SectionType === SectionType.QNASECTION) { - const startLine = section.ParseTree.start.line - 1 + const startLine = section.Range.Start.Line - 1; let stopLine if (index + 1 < sections.length) { - stopLine = sections[index + 1].ParseTree.start.line - 1 - if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine >= stopLine || originList.Length <= stopLine) { + stopLine = sections[index + 1].Range.Start.Line - 1 + if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine) { throw new Error("index out of range.") } } else { stopLine = originList.length } + section.Range.End.Line = stopLine; let destList if (section.SectionType === SectionType.QNASECTION) { @@ -329,15 +359,10 @@ class LUParser { } section.Body = destList.join(NEWLINE) - section.StartLine = startLine - section.StopLine = stopLine - 1 if (section.SectionType === SectionType.NESTEDINTENTSECTION) { LUParser.extractSectionBody(section.SimpleIntentSections, originList.slice(0, stopLine).join(NEWLINE)) } - } else { - section.StartLine = section.ParseTree.start.line - section.StopLine = section.ParseTree.stop.line - 1 } }) } diff --git a/packages/lu/src/parser/lufile/luResource.js b/packages/lu/src/parser/lufile/luResource.js index 73196d96e..f1d752d62 100644 --- a/packages/lu/src/parser/lufile/luResource.js +++ b/packages/lu/src/parser/lufile/luResource.js @@ -5,9 +5,9 @@ class LUResource { * @param {any[]} errors */ constructor(sections, content, errors) { - this.Sections = sections; + this.Sections = sections || []; this.Content = content; - this.Errors = errors; + this.Errors = errors || []; } } diff --git a/packages/lu/src/parser/lufile/modelInfoSection.js b/packages/lu/src/parser/lufile/modelInfoSection.js index b380d0429..bc0fc4e26 100644 --- a/packages/lu/src/parser/lufile/modelInfoSection.js +++ b/packages/lu/src/parser/lufile/modelInfoSection.js @@ -1,17 +1,23 @@ const ModelInfoSectionContext = require('./generated/LUFileParser').LUFileParser.ModelInfoSectionContext; const LUSectionTypes = require('./../utils/enums/lusectiontypes'); +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class LUModelInfo { +class LUModelInfo extends BaseSection { /** * * @param {ModelInfoSectionContext} parseTree */ constructor(parseTree) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.MODELINFOSECTION; this.ModelInfo = parseTree.modelInfoDefinition().getText(); this.Errors = []; this.Id = `${this.SectionType}_${this.ModelInfo}`; + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } } diff --git a/packages/lu/src/parser/lufile/nestedIntentSection.js b/packages/lu/src/parser/lufile/nestedIntentSection.js index b072ca667..593849489 100644 --- a/packages/lu/src/parser/lufile/nestedIntentSection.js +++ b/packages/lu/src/parser/lufile/nestedIntentSection.js @@ -2,14 +2,17 @@ const NestedIntentSectionContext = require('./generated/LUFileParser').LUFilePar const SimpleIntentSection = require('./simpleIntentSection'); const LUSectionTypes = require('./../utils/enums/lusectiontypes'); const NEWLINE = require('os').EOL; +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class NestedIntentSection { +class NestedIntentSection extends BaseSection { /** * * @param {NestedIntentSectionContext} parseTree */ constructor(parseTree, content) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.NESTEDINTENTSECTION; this.Name = this.ExtractName(parseTree); this.Body = this.ExtractBody(parseTree, content); @@ -22,6 +25,9 @@ class NestedIntentSection { } this.Id = `${this.SectionType}_${this.Name}`; + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } ExtractName(parseTree) { @@ -29,8 +35,8 @@ class NestedIntentSection { } ExtractBody(parseTree, content) { - const startLine = parseTree.start.line - 1 - const stopLine = parseTree.stop.line - 1 + const startLine = parseTree.start.line - 1; + const stopLine = parseTree.stop.line - 1; const originList = content.split(/\r?\n/) if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine || originList.Length <= stopLine) { throw new Error("index out of range.") diff --git a/packages/lu/src/parser/lufile/newEntitySection.js b/packages/lu/src/parser/lufile/newEntitySection.js index 2e0fd0561..84b857706 100644 --- a/packages/lu/src/parser/lufile/newEntitySection.js +++ b/packages/lu/src/parser/lufile/newEntitySection.js @@ -3,14 +3,17 @@ const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity; const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; const LUSectionTypes = require('./../utils/enums/lusectiontypes'); const InvalidCharsInIntentOrEntityName = require('./../utils/enums/invalidchars').InvalidCharsInIntentOrEntityName; +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class NewEntitySection { +class NewEntitySection extends BaseSection { /** * * @param {NewEntitySectionContext} parseTree */ constructor(parseTree) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.NEWENTITYSECTION; this.Errors = [] this.Name = this.ExtractName(parseTree); @@ -21,6 +24,9 @@ class NewEntitySection { this.RegexDefinition = this.ExtractRegexDefinition(parseTree); this.ListBody = this.ExtractSynonymsOrPhraseList(parseTree); this.Id = `${this.SectionType}_${this.Name}`; + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } ExtractName(parseTree) { diff --git a/packages/lu/src/parser/lufile/parseFileContents.js b/packages/lu/src/parser/lufile/parseFileContents.js index 68d701bc2..3a974f823 100644 --- a/packages/lu/src/parser/lufile/parseFileContents.js +++ b/packages/lu/src/parser/lufile/parseFileContents.js @@ -369,9 +369,9 @@ const validateNDepthEntities = function(collection, entitiesAndRoles, intentsCol * @param {String} srcItemName * @param {String} tgtFeatureType * @param {String} tgtFeatureName - * @param {String} line + * @param {Range} range */ -const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureType, tgtFeatureName, line) { +const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureType, tgtFeatureName, range) { switch(srcItemType) { case INTENTTYPE: case EntityTypeEnum.SIMPLE: @@ -382,7 +382,7 @@ const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureT let errorMsg = `'patternany' entity cannot be added as a feature. Invalid definition found for "@ ${srcItemType} ${srcItemName} usesFeature ${tgtFeatureName}"`; let error = BuildDiagnostic({ message: errorMsg, - context: line + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -392,7 +392,7 @@ const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureT let errorMsg = `Invalid definition found for "@ ${srcItemType} ${srcItemName} usesFeature ${tgtFeatureName}". usesFeature is only available for intent, ${plAllowedTypes.join(', ')}`; let error = BuildDiagnostic({ message: errorMsg, - context: line + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); break; @@ -403,16 +403,16 @@ const validateFeatureAssignment = function(srcItemType, srcItemName, tgtFeatureT * @param {Object} tgtItem * @param {String} feature * @param {String} featureType - * @param {Object} line + * @param {Object} range */ -const addFeatures = function(tgtItem, feature, featureType, line, featureProperties) { +const addFeatures = function(tgtItem, feature, featureType, range, featureProperties) { // target item cannot have the same name as the feature name if (tgtItem.name === feature) { // Item must be defined before being added as a feature. let errorMsg = `Source and target cannot be the same for usesFeature. e.g. x usesFeature x is invalid. "${tgtItem.name}" usesFeature "${feature}" is invalid.`; let error = BuildDiagnostic({ message: errorMsg, - context: line + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -452,7 +452,7 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) { let errorMsg = `Intents can only have usesFeature and nothing else. Invalid definition for "${section.Name}".`; let error = BuildDiagnostic({ message: errorMsg, - context: section.ParseTree.newEntityDefinition().newEntityLine() + range: section.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -469,26 +469,26 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) { if (entityExists) { if (entityExists.type === EntityTypeEnum.PHRASELIST) { // de-dupe and add features to intent. - validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(intentExists, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.phraseListFeature); + validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.Range); + addFeatures(intentExists, feature, featureTypeEnum.featureToModel, section.Range, featureProperties.phraseListFeature); // set enabledForAllModels on this phrase list let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature); if (plEnity.enabledForAllModels === undefined) plEnity.enabledForAllModels = false; } else { // de-dupe and add model to intent. - validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.entityFeatureToModel[entityExists.type]); + validateFeatureAssignment(section.Type, section.Name, entityExists.type, feature, section.Range); + addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.Range, featureProperties.entityFeatureToModel[entityExists.type]); } } else if (featureIntentExists) { // Add intent as a feature to another intent - validateFeatureAssignment(section.Type, section.Name, INTENTTYPE, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.intentFeatureToModel); + validateFeatureAssignment(section.Type, section.Name, INTENTTYPE, feature, section.Range); + addFeatures(intentExists, feature, featureTypeEnum.modelToFeature, section.Range, featureProperties.intentFeatureToModel); } else { // Item must be defined before being added as a feature. let errorMsg = `Features must be defined before assigned to an intent. No definition found for feature "${feature}" in usesFeature definition for intent "${section.Name}"`; let error = BuildDiagnostic({ message: errorMsg, - context: section.ParseTree.newEntityDefinition().newEntityLine() + range: section.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -497,7 +497,7 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) { let errorMsg = `Features can only be added to intents that have a definition. Invalid feature definition found for intent "${section.Name}".`; let error = BuildDiagnostic({ message: errorMsg, - context: section.ParseTree.newEntityDefinition().newEntityLine() + range: section.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -517,26 +517,26 @@ const parseFeatureSections = function(parsedContent, featuresToProcess) { if (featureExists) { if (featureExists.type === EntityTypeEnum.PHRASELIST) { // de-dupe and add features to intent. - validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(srcEntity, feature, featureTypeEnum.featureToModel, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.phraseListFeature); + validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.Range); + addFeatures(srcEntity, feature, featureTypeEnum.featureToModel, section.Range, featureProperties.phraseListFeature); // set enabledForAllModels on this phrase list let plEnity = parsedContent.LUISJsonStructure.model_features.find(item => item.name == feature); if (plEnity.enabledForAllModels === undefined) plEnity.enabledForAllModels = false; } else { // de-dupe and add model to intent. - validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.entityFeatureToModel[featureExists.type]); + validateFeatureAssignment(entityType, section.Name, featureExists.type, feature, section.Range); + addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.Range, featureProperties.entityFeatureToModel[featureExists.type]); } } else if (featureIntentExists) { // Add intent as a feature to another intent - validateFeatureAssignment(entityType, section.Name, INTENTTYPE, feature, section.ParseTree.newEntityDefinition().newEntityLine()); - addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.ParseTree.newEntityDefinition().newEntityLine(), featureProperties.intentFeatureToModel); + validateFeatureAssignment(entityType, section.Name, INTENTTYPE, feature, section.Range); + addFeatures(srcEntity, feature, featureTypeEnum.modelToFeature, section.Range, featureProperties.intentFeatureToModel); } else { // Item must be defined before being added as a feature. let errorMsg = `Features must be defined before assigned to an entity. No definition found for feature "${feature}" in usesFeature definition for entity "${section.Name}"`; let error = BuildDiagnostic({ message: errorMsg, - context: section.ParseTree.newEntityDefinition().newEntityLine() + range: section.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -641,7 +641,7 @@ const parseAndHandleImportSection = async function (parsedContent, luResource) { let errorMsg = `URI: "${linkValue}" appears to be invalid. Please double check the URI or re-try this parse when you are connected to the internet.`; let error = BuildDiagnostic({ message: errorMsg, - context: luImport.ParseTree + range: luImport.Range }) throw (new exception(retCode.errorCode.INVALID_URI, error.toString(), [error])); @@ -651,7 +651,7 @@ const parseAndHandleImportSection = async function (parsedContent, luResource) { let errorMsg = `URI: "${linkValue}" appears to be invalid. Please double check the URI or re-try this parse when you are connected to the internet.`; let error = BuildDiagnostic({ message: errorMsg, - context: luImport.ParseTree + range: luImport.Range }) throw (new exception(retCode.errorCode.INVALID_URI, error.toString(), [error])); @@ -731,8 +731,8 @@ const parseAndHandleNestedIntentSection = function (luResource, enableMergeInten sections.forEach(section => { if (enableMergeIntents) { let mergedIntentSection = section.SimpleIntentSections[0]; - mergedIntentSection.ParseTree = section.ParseTree; mergedIntentSection.Name = section.Name; + mergedIntentSection.Range = section.Range; for (let idx = 1; idx < section.SimpleIntentSections.length; idx++) { mergedIntentSection.UtteranceAndEntitiesMap = mergedIntentSection.UtteranceAndEntitiesMap.concat(section.SimpleIntentSections[idx].UtteranceAndEntitiesMap); mergedIntentSection.Entities = mergedIntentSection.Entities.concat(section.SimpleIntentSections[idx].Entities); @@ -783,10 +783,10 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { utterance = handleAtForPattern(utterance, entitiesFound, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles); let mixedEntity = entitiesFound.filter(item => item.type != LUISObjNameEnum.PATTERNANYENTITY); if (mixedEntity.length !== 0) { - let errorMsg = `Utterance "${utteranceAndEntities.context.getText()}" has mix of entites with labelled values and ones without. Please update utterance to either include labelled values for all entities or remove labelled values from all entities.`; + let errorMsg = `Utterance "${utteranceAndEntities.contextText}" has mix of entites with labelled values and ones without. Please update utterance to either include labelled values for all entities or remove labelled values from all entities.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -795,10 +795,10 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { let prebuiltEntities = entitiesFound.filter(item => builtInTypes.consolidatedList.includes(item.entity)); prebuiltEntities.forEach(prebuiltEntity => { if (parsedContent.LUISJsonStructure.prebuiltEntities.findIndex(e => e.name === prebuiltEntity.entity) < 0) { - let errorMsg = `Pattern "${utteranceAndEntities.context.getText()}" has prebuilt entity ${prebuiltEntity.entity}. Please define it explicitly with @ prebuilt ${prebuiltEntity.entity}.`; + let errorMsg = `Pattern "${utteranceAndEntities.contextText}" has prebuilt entity ${prebuiltEntity.entity}. Please define it explicitly with @ prebuilt ${prebuiltEntity.entity}.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -865,7 +865,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { let errorMsg = `Utterance "${utterance}" has invalid reference to Phrase List entity "${nonAllowedPhrseListEntityInUtterance.name}". Phrase list entities cannot be given an explicit labelled value.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }); throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -897,7 +897,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }); throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -911,7 +911,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }); throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -925,7 +925,7 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }); throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -966,10 +966,10 @@ const parseAndHandleSimpleIntentSection = function (parsedContent, luResource) { } entitiesFound.forEach(item => { if (item.startPos > item.endPos) { - let errorMsg = `No labelled value found for entity: "${item.entity}" in utterance: "${utteranceAndEntities.context.getText()}"`; + let errorMsg = `No labelled value found for entity: "${item.entity}" in utterance: "${utteranceAndEntities.contextText}"`; let error = BuildDiagnostic({ message: errorMsg, - context: utteranceAndEntities.context + range: utteranceAndEntities.range }) throw (new exception(retCode.errorCode.MISSING_LABELLED_VALUE, error.toString(), [error])); @@ -1026,10 +1026,10 @@ const getEntityType = function(entityName, entities) { * Helper function to validate that new roles being added are unique at the application level. * @param {Object} parsedContent with that contains list of additional files to parse, parsed LUIS object and parsed QnA object * @param {String[]} roles string array of new roles to be added - * @param {String} line current line being parsed. + * @param {String} range range * @param {String} entityName name of the entity being added. */ -const validateAndGetRoles = function(parsedContent, roles, line, entityName, entityType) { +const validateAndGetRoles = function(parsedContent, roles, range, entityName, entityType) { let newRoles = roles ? roles.split(',').map(item => item.replace(/[\'\"]/g, "").trim()) : []; // de-dupe roles newRoles = [...new Set(newRoles)]; @@ -1037,7 +1037,7 @@ const validateAndGetRoles = function(parsedContent, roles, line, entityName, ent if(parsedContent.LUISJsonStructure.flatListOfEntityAndRoles) { // Duplicate entity names are not allowed // Entity name cannot be same as a role name - verifyUniqueEntityName(parsedContent, entityName, entityType, line); + verifyUniqueEntityName(parsedContent, entityName, entityType, range); newRoles.forEach(role => { let roleFound = parsedContent.LUISJsonStructure.flatListOfEntityAndRoles.find(item => item.roles.includes(role) || item.name === role); @@ -1048,7 +1048,7 @@ const validateAndGetRoles = function(parsedContent, roles, line, entityName, ent let errorMsg = `Roles must be unique across entity types. Invalid role definition found "${entityName}". Prior definition - '@ ${roleFound.type} ${roleFound.name}${roleFound.roles.length > 0 ? ` hasRoles ${roleFound.roles.join(',')}` : ``}'`; let error = BuildDiagnostic({ message: errorMsg, - context: line + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -1103,7 +1103,7 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale) let errorMsg = `No type definition found for entity "${entityName}". Supported types are ${Object.values(EntityTypeEnum).join(', ')}. Note: Type names are case sensitive.`; let error = BuildDiagnostic({ message: errorMsg, - context: entity.ParseTree.newEntityDefinition().newEntityLine() + range: entity.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); }; @@ -1112,12 +1112,12 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale) let errorMsg = `Entity name "${entityName}" cannot be the same as entity type "${entityType}"`; let error = BuildDiagnostic({ message: errorMsg, - context: entity.ParseTree.newEntityDefinition().newEntityLine() + range: entity.Range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } - let entityRoles = validateAndGetRoles(parsedContent, entity.Roles, entity.ParseTree.newEntityDefinition().newEntityLine(), entityName, entityType); - let PAEntityRoles = RemoveDuplicatePatternAnyEntity(parsedContent, entityName, entityType, entity.ParseTree.newEntityDefinition().newEntityLine()); + let entityRoles = validateAndGetRoles(parsedContent, entity.Roles, entity.Range, entityName, entityType); + let PAEntityRoles = RemoveDuplicatePatternAnyEntity(parsedContent, entityName, entityType, entity.Range); if (PAEntityRoles.length > 0) { PAEntityRoles.forEach(role => { if (!entityRoles.includes(role)) entityRoles.push(role); @@ -1125,7 +1125,7 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale) } switch(entityType) { case EntityTypeEnum.ML: - handleNDepthEntity(parsedContent, entityName, entityRoles, entity.ListBody, entity.ParseTree.newEntityDefinition().newEntityLine()); + handleNDepthEntity(parsedContent, entityName, entityRoles, entity.ListBody, entity.Range); break; case EntityTypeEnum.SIMPLE: addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles); @@ -1140,28 +1140,28 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale) line.trim().substr(1).trim().replace(/[\[\]]/g, '').split(/[,;]/g).map(item => item.trim()).forEach(item => candidateChildren.push(item)); }) } - handleComposite(parsedContent, entityName,`[${candidateChildren.join(',')}]`, entityRoles, entity.ParseTree.newEntityDefinition().newEntityLine(), false, entity.Type !== undefined); + handleComposite(parsedContent, entityName,`[${candidateChildren.join(',')}]`, entityRoles, entity.Range, false, entity.Type !== undefined); break; case EntityTypeEnum.LIST: - handleClosedList(parsedContent, entityName, entity.ListBody.map(item => item.trim()), entityRoles, entity.ParseTree.newEntityDefinition().newEntityLine()); + handleClosedList(parsedContent, entityName, entity.ListBody.map(item => item.trim()), entityRoles, entity.Range); break; case EntityTypeEnum.PATTERNANY: - handlePatternAny(parsedContent, entityName, entityRoles, entity.ParseTree.newEntityDefinition().newEntityLine()); + handlePatternAny(parsedContent, entityName, entityRoles, entity.Range); break; case EntityTypeEnum.PREBUILT: - handlePrebuiltEntity(parsedContent, 'prebuilt', entityName, entityRoles, locale, log, entity.ParseTree.newEntityDefinition().newEntityLine()); + handlePrebuiltEntity(parsedContent, 'prebuilt', entityName, entityRoles, locale, log, entity.Range); break; case EntityTypeEnum.REGEX: if (entity.ListBody[0]) { - handleRegExEntity(parsedContent, entityName, entity.ListBody[0].trim().substr(1).trim(), entityRoles, entity.ParseTree.newEntityDefinition().newEntityLine()); + handleRegExEntity(parsedContent, entityName, entity.ListBody[0].trim().substr(1).trim(), entityRoles, entity.Range); } else { - handleRegExEntity(parsedContent, entityName, entity.RegexDefinition, entityRoles, entity.ParseTree.newEntityDefinition().newEntityLine()); + handleRegExEntity(parsedContent, entityName, entity.RegexDefinition, entityRoles, entity.Range); } break; case EntityTypeEnum.ML: break; case EntityTypeEnum.PHRASELIST: - handlePhraseList(parsedContent, entityName, entity.Type, entityRoles, entity.ListBody.map(item => item.trim().substr(1).trim()), entity.ParseTree.newEntityDefinition().newEntityLine()); + handlePhraseList(parsedContent, entityName, entity.Type, entityRoles, entity.ListBody.map(item => item.trim().substr(1).trim()), entity.Range); default: //Unknown entity type break; @@ -1183,14 +1183,14 @@ const parseAndHandleEntityV2 = function (parsedContent, luResource, log, locale) * @param {String} entityName * @param {String[]} entityRoles * @param {String[]} entityLines - * @param {Object} line + * @param {Object} range */ -const handleNDepthEntity = function(parsedContent, entityName, entityRoles, entityLines, line) { +const handleNDepthEntity = function(parsedContent, entityName, entityRoles, entityLines, range) { const SPACEASTABS = 4; addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles); let rootEntity = parsedContent.LUISJsonStructure.entities.find(item => item.name == entityName); rootEntity.explicitlyAdded = true; - let defLine = line.start.line; + let defLine = range.Start.Line let baseTabLevel = 0; let entityIdxByLevel = []; let currentParentEntity = undefined; @@ -1214,7 +1214,7 @@ const handleNDepthEntity = function(parsedContent, entityName, entityRoles, enti // Get current tab level let tabLevel = Math.ceil(groupsFound.groups.leadingSpaces !== undefined ? groupsFound.groups.leadingSpaces.length / SPACEASTABS : 0) || (groupsFound.groups.leadingTabs !== undefined ? groupsFound.groups.leadingTabs.length : 0); - if (defLine === line.start.line + 1) { + if (defLine === range.Start.Line + 1) { // remember the tab level at the first line of child definition baseTabLevel = tabLevel; // Push the ID of the parent since we are proessing the first child entity @@ -1269,10 +1269,10 @@ const pushNDepthChild = function(entity, childEntityName, context, childFeatures * @param {Object} parsedContent * @param {String} entityName * @param {String} entityType - * @param {Object} line + * @param {Object} range * @param {Boolean} checkTypesAlignment */ -const verifyUniqueEntityName = function(parsedContent, entityName, entityType, line, checkTypesAlignment) { +const verifyUniqueEntityName = function(parsedContent, entityName, entityType, range, checkTypesAlignment) { // Duplicate entity names are not allowed // Entity name cannot be same as a role name let matchType = ""; @@ -1294,7 +1294,7 @@ const verifyUniqueEntityName = function(parsedContent, entityName, entityType, l let errorMsg = `${matchType} Prior definition - '@ ${entityFound.type} ${entityFound.name}${entityFound.roles.length > 0 ? ` hasRoles ${entityFound.roles.join(',')}` : ``}'`; let error = BuildDiagnostic({ message: errorMsg, - context: line + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -1330,9 +1330,9 @@ const handlePatternAny = function(parsedContent, entityName, entityRoles) { * @param {Object} parsedContent Object containing current parsed content - LUIS, QnA, QnA alterations. * @param {String} pEntityName name of entity * @param {String} entityType type of entity - * @param {String} entityLine current line being parsed + * @param {String} range range */ -const RemoveDuplicatePatternAnyEntity = function(parsedContent, pEntityName, entityType, entityLine) { +const RemoveDuplicatePatternAnyEntity = function(parsedContent, pEntityName, entityType, range) { // see if we already have this as Pattern.Any entity // see if we already have this in patternAny entity collection; if so, remove it but remember the roles (if any) let PAIdx = -1; @@ -1350,7 +1350,7 @@ const RemoveDuplicatePatternAnyEntity = function(parsedContent, pEntityName, ent let errorMsg = `Phrase lists cannot be used as an entity in a pattern "${pEntityName}"`; let error = BuildDiagnostic({ message: errorMsg, - context: entityLine + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); } @@ -1368,7 +1368,7 @@ const RemoveDuplicatePatternAnyEntity = function(parsedContent, pEntityName, ent * @param {String []} entityRoles Array of roles * @param {String []} valuesList Array of individual lines to be processed and added to phrase list. */ -const handlePhraseList = function(parsedContent, entityName, entityType, entityRoles, valuesList, currentLine) { +const handlePhraseList = function(parsedContent, entityName, entityType, entityRoles, valuesList, range) { let isPLEnabledForAllModels = undefined; let isPLEnabled = undefined; if (entityRoles.length !== 0) { @@ -1421,7 +1421,7 @@ const handlePhraseList = function(parsedContent, entityName, entityType, entityR let errorMsg = `Phrase list: "${entityName}" has conflicting definitions. One marked interchangeable and another not interchangeable`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -1447,9 +1447,9 @@ const handlePhraseList = function(parsedContent, entityName, entityType, entityR * @param {String []} entityRoles list of entity roles * @param {String} locale current locale * @param {Boolean} log boolean to indicate if errors should be sent to stdout - * @param {String} currentLine current line being parsed. + * @param {String} range range */ -const handlePrebuiltEntity = function(parsedContent, entityName, entityType, entityRoles, locale, log, currentLine) { +const handlePrebuiltEntity = function(parsedContent, entityName, entityType, entityRoles, locale, log, range) { locale = locale ? locale.toLowerCase() : 'en-us'; // check if this pre-built entity is already labelled in an utterance and or added as a simple entity. if so, throw an error. try { @@ -1465,7 +1465,7 @@ const handlePrebuiltEntity = function(parsedContent, entityName, entityType, ent let errorMsg = `Unknown PREBUILT entity '${entityType}'. Available pre-built types are ${builtInTypes.consolidatedList.join(',')}`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -1479,7 +1479,7 @@ const handlePrebuiltEntity = function(parsedContent, entityName, entityType, ent let errorMsg = `PREBUILT entity '${entityType}' is not available for the requested locale '${locale}'`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString(), [error])); @@ -1502,11 +1502,11 @@ const handlePrebuiltEntity = function(parsedContent, entityName, entityType, ent * @param {String} entityName entity name * @param {String} entityType entity type * @param {String []} entityRoles collection of roles - * @param {String} currentLine current line being parsed + * @param {String} range range * @param {Boolean} inlineChildRequired boolean to indicate if children definition must be defined inline. * @param {Boolean} isEntityTypeDefinition Type definition is included */ -const handleComposite = function(parsedContent, entityName, entityType, entityRoles, currentLine, inlineChildRequired, isEntityTypeDefinition) { +const handleComposite = function(parsedContent, entityName, entityType, entityRoles, range, inlineChildRequired, isEntityTypeDefinition) { // remove simple entity definitions for composites but carry forward roles. // Find this entity if it exists in the simple entity collection let simpleEntityExists = (parsedContent.LUISJsonStructure.entities || []).find(item => item.name == entityName); @@ -1527,7 +1527,7 @@ const handleComposite = function(parsedContent, entityName, entityType, entityRo let errorMsg = `Composite entity: ${entityName} is missing child entity definitions. Child entities are denoted via [entity1, entity2] notation.`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.INVALID_COMPOSITE_ENTITY, error.toString(), [error])); @@ -1548,7 +1548,7 @@ const handleComposite = function(parsedContent, entityName, entityType, entityRo let errorMsg = `Composite entity: ${entityName} has multiple definition with different children. \n 1. ${compositeChildren.join(', ')}\n 2. ${compositeEntity.children.join(', ')}`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.INVALID_COMPOSITE_ENTITY, error.toString(), [error])); @@ -1568,9 +1568,9 @@ const handleComposite = function(parsedContent, entityName, entityType, entityRo * @param {String} entityName entity name * @param {String []} listLines lines to parse for the list entity * @param {String []} entityRoles collection of roles found - * @param {String} currentLine current line being parsed. + * @param {String} range range */ -const handleClosedList = function (parsedContent, entityName, listLines, entityRoles, currentLine) { +const handleClosedList = function (parsedContent, entityName, listLines, entityRoles, range) { // check if this list entity is already labelled in an utterance and or added as a simple entity. if so, throw an error. try { let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityName, 'List'); @@ -1614,7 +1614,7 @@ const handleClosedList = function (parsedContent, entityName, listLines, entityR let errorMsg = `Closed list ${entityName} has synonyms list "${line}" without a normalized value.`; let error = BuildDiagnostic({ message: errorMsg, - context: currentLine + range: range }) throw (new exception(retCode.errorCode.SYNONYMS_NOT_A_LIST, error.toString(), [error])); @@ -1657,7 +1657,7 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo entityType = parsedRoleAndType.entityType; let pEntityName = (entityName.toLowerCase() === 'prebuilt') ? entityType : entityName; - let PAEntityRoles = RemoveDuplicatePatternAnyEntity(parsedContent, pEntityName, entityType, entity.ParseTree.entityDefinition().entityLine()); + let PAEntityRoles = RemoveDuplicatePatternAnyEntity(parsedContent, pEntityName, entityType, entity.Range); if (PAEntityRoles.length > 0) { PAEntityRoles.forEach(role => { if (!entityRoles.includes(role)) entityRoles.push(role); @@ -1667,7 +1667,7 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo // add this entity to appropriate place // is this a builtin type? if (builtInTypes.consolidatedList.includes(entityType)) { - handlePrebuiltEntity(parsedContent, entityName, entityType, entityRoles, locale, log, entity.ParseTree.entityDefinition().entityLine()); + handlePrebuiltEntity(parsedContent, entityName, entityType, entityRoles, locale, log, entity.Range); } else if (entityType.toLowerCase() === 'simple') { // add this to entities if it doesnt exist addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles); @@ -1684,7 +1684,7 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo let errorMsg = `QnA alteration section: "${alterationlist}" does not have list decoration. Prefix line with "-" or "+" or "*"`; let error = BuildDiagnostic({ message: errorMsg, - context: entity.ParseTree.entityDefinition().entityLine() + range: entity.range }) throw (new exception(retCode.errorCode.SYNONYMS_NOT_A_LIST, error.toString(), [error])); @@ -1732,17 +1732,17 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo if (entityType.toLowerCase().includes('interchangeable')) { entityName += '(interchangeable)'; } - handlePhraseList(parsedContent, entityName, entityType, entityRoles, entity.SynonymsOrPhraseList, entity.ParseTree.entityDefinition().entityLine()); + handlePhraseList(parsedContent, entityName, entityType, entityRoles, entity.SynonymsOrPhraseList, entity.Range); } else if (entityType.startsWith('[')) { - handleComposite(parsedContent, entityName, entityType, entityRoles, entity.ParseTree.entityDefinition().entityLine(), true, true); + handleComposite(parsedContent, entityName, entityType, entityRoles, entity.Range, true, true); } else if (entityType.startsWith('/')) { if (entityType.endsWith('/')) { - handleRegExEntity(parsedContent, entityName, entityType, entityRoles, entity.ParseTree.entityDefinition().entityLine()); + handleRegExEntity(parsedContent, entityName, entityType, entityRoles, entity.Range); } else { let errorMsg = `RegEx entity: ${regExEntity.name} is missing trailing '/'. Regex patterns need to be enclosed in forward slashes. e.g. /[0-9]/`; let error = BuildDiagnostic({ message: errorMsg, - context: entity.ParseTree.entityDefinition().entityLine() + range: entity.Range }) throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString(), [error])); @@ -1759,9 +1759,9 @@ const parseAndHandleEntitySection = function (parsedContent, luResource, log, lo * @param {String} entityName name of entity * @param {String} entityType type of entity * @param {String []} entityRoles array of entity roles found - * @param {String} entityLine current line being parsed/ handled. + * @param {String} range range */ -const handleRegExEntity = function(parsedContent, entityName, entityType, entityRoles, entityLine) { +const handleRegExEntity = function(parsedContent, entityName, entityType, entityRoles, range) { // check if this regex entity is already labelled in an utterance and or added as a simple entity. if so, throw an error. try { let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityName, 'RegEx'); @@ -1779,7 +1779,7 @@ const handleRegExEntity = function(parsedContent, entityName, entityType, entity let errorMsg = `RegEx entity: ${entityName} has empty regex pattern defined.`; let error = BuildDiagnostic({ message: errorMsg, - context: entityLine + range: range }) throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString(), [error])); @@ -1796,7 +1796,7 @@ const handleRegExEntity = function(parsedContent, entityName, entityType, entity let errorMsg = `RegEx entity: ${regExEntity.name} has multiple regex patterns defined. \n 1. /${regex}/\n 2. /${regExEntity.regexPattern}/`; let error = BuildDiagnostic({ message: errorMsg, - context: entityLine + range: range }) throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString(), [error])); diff --git a/packages/lu/src/parser/lufile/qnaSection.js b/packages/lu/src/parser/lufile/qnaSection.js index ff35dc012..947ab730a 100644 --- a/packages/lu/src/parser/lufile/qnaSection.js +++ b/packages/lu/src/parser/lufile/qnaSection.js @@ -2,14 +2,17 @@ const QnaSectionContext = require('./generated/LUFileParser').LUFileParser.QnaSe const LUSectionTypes = require('./../utils/enums/lusectiontypes'); const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; const QNA_GENERIC_SOURCE = "custom editorial"; +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class QnaSection { +class QnaSection extends BaseSection { /** * * @param {QnaSectionContext} parseTree */ constructor(parseTree) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.QNASECTION; this.Questions = [this.ExtractQuestion(parseTree)]; let result = this.ExtractMoreQuestions(parseTree); @@ -25,6 +28,9 @@ class QnaSection { this.Errors = this.Errors.concat(result.errors); this.QAPairId = this.ExtractAssignedId(parseTree); this.source = this.ExtractSourceInfo(parseTree); + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } ExtractSourceInfo(parseTree) { diff --git a/packages/lu/src/parser/lufile/sectionOperator.js b/packages/lu/src/parser/lufile/sectionOperator.js index 559e5ee9c..65152f57d 100644 --- a/packages/lu/src/parser/lufile/sectionOperator.js +++ b/packages/lu/src/parser/lufile/sectionOperator.js @@ -1,73 +1,275 @@ const luParser = require('./luParser'); const helpers = require('./../utils/helpers'); const NEWLINE = require('os').EOL; - +const LUResource = require('./luResource'); +const LUSectionTypes = require('../utils/enums/lusectiontypes'); +const {cloneDeepWith, cloneDeep, isFunction} = require('lodash'); + class SectionOperator { - + /** * @param {LUResource} luresource */ constructor(luresource) { - this.Luresource = luresource; + const {Sections, Errors, Content } = cloneDeepWith(luresource, (value) => { + return isFunction(value) ? undfined: cloneDeep(value); + }); + this.Luresource = new LUResource(Sections, Content, Errors); } // After CRUD, section Ids will keep same unless you change section name. addSection(sectionContent) { sectionContent = helpers.sanitizeNewLines(sectionContent); - const newContent = this.Luresource.Content !== '' ? `${this.Luresource.Content}${NEWLINE}${sectionContent}` : sectionContent; - const result = luParser.parse(newContent); - return result; + const newResource = luParser.parseWithRef(sectionContent, this.Luresource); + if (!newResource) { + return this.Luresource; + } + + if (this.Luresource.Sections.some(u => u.Id === newResource.Id)) { + throw new Error(`Section with id: ${newResource.Id} exists.`); + } + + const offset = !this.Luresource.Content ? 0 : this.Luresource.Content.split(/\r?\n/).length; + + this.Luresource.Content = this.Luresource.Content !== '' ? `${this.Luresource.Content}${NEWLINE}${sectionContent}` : sectionContent; + + // add a NestedIntentSection may appears multiple sections + this.adjustRangeForAddSection(newResource.Sections, offset); + this.Luresource.Sections.push(...newResource.Sections); + + this.adjustRangeForErrors(newResource.Errors, offset); + this.Luresource.Errors.push(...newResource.Errors); + + luParser.extractSectionBody(this.Luresource.Sections, this.Luresource.Content); + if (this.Luresource.hashATNConfig) { + console.log('----------------------------'); + } + return this.Luresource; } updateSection(id, sectionContent) { sectionContent = helpers.sanitizeNewLines(sectionContent); - const section = this.Luresource.Sections.find(u => u.Id === id); - if (!section) { + const sectionIndex = this.Luresource.Sections.findIndex(u => u.Id === id); + if (sectionIndex < 0) { + return this.Luresource; + } + + const oldSection = this.Luresource.Sections[sectionIndex]; + const newResource = luParser.parseWithRef(sectionContent, this.Luresource); + if (!newResource) { return this.Luresource; } - const startLine = section.StartLine; - const stopLine = section.StopLine; + // add a NestedIntentSection may appears multiple sections + const startLine = oldSection.Range.Start.Line; + const endLine = oldSection.Range.End.Line; - const newContent = this.replaceRangeContent(this.Luresource.Content, startLine, stopLine, sectionContent); + this.removeErrors(this.Luresource.Errors, startLine, endLine); - return luParser.parse(newContent); + // adjust original errors + const newLineRange = sectionContent.split(/\r?\n/).length; + const originalRange = endLine - startLine + 1; + this.adjustRangeForErrors(this.Luresource.Errors, newLineRange - originalRange, endLine); + + // adjust updated sections' errors + const offset = oldSection.Range.Start.Line - newResource.Sections[0].Range.Start.Line; + this.adjustRangeForErrors(newResource.Errors, offset); + this.Luresource.Errors.push(...newResource.Errors); + + this.Luresource.Content = this.replaceRangeContent(this.Luresource.Content, oldSection.Range.Start.Line - 1, oldSection.Range.End.Line - 1, sectionContent); + this.adjustRangeForUpdateSection(sectionIndex, newResource.Sections); + + luParser.extractSectionBody(this.Luresource.Sections, this.Luresource.Content); + if (this.Luresource.hashATNConfig) { + console.log('----------------------------'); + } + return this.Luresource; } deleteSection(id) { - const section = this.Luresource.Sections.find(u => u.Id === id); - if (!section) { + const sectionIndex = this.Luresource.Sections.findIndex(u => u.Id === id); + if (sectionIndex < 0) { return this; } - const startLine = section.StartLine; - const stopLine = section.StopLine; + const oldSection = this.Luresource.Sections[sectionIndex]; + const startLine = oldSection.Range.Start.Line; + const endLine = oldSection.Range.End.Line; - const newContent = this.replaceRangeContent(this.Luresource.Content, startLine, stopLine, undefined); + this.removeErrors(this.Luresource.Errors, startLine, endLine); + this.adjustRangeForErrors(this.Luresource.Errors, startLine - endLine, endLine); - return luParser.parse(newContent); + this.Luresource.Sections.splice(sectionIndex, 1); + this.Luresource.Content = this.replaceRangeContent(this.Luresource.Content, startLine - 1, endLine - 1, undefined); + + const offset = endLine - startLine + 1; + this.adjustRangeForDeleteSection(sectionIndex, offset); + luParser.extractSectionBody(this.Luresource.Sections, this.Luresource.Content); + if (this.Luresource.hashATNConfig) { + console.log('----------------------------'); + } + return this.Luresource; } insertSection(id, sectionContent) { + // insert into the front of the old section sectionContent = helpers.sanitizeNewLines(sectionContent); - const section = this.Luresource.Sections.find(u => u.Id === id); - if (!section && this.Luresource.Sections.length > 0 ) { + const sectionIndex = this.Luresource.Sections.findIndex(u => u.Id === id); + + if (sectionIndex < 0 && this.Luresource.Sections.length > 0) { + return this.Luresource; + } + + // if secionIndex < 0 and the luresource is empty, just add it + if (sectionIndex < 0 && this.Luresource.Sections.length === 0) { + return this.addSection(sectionContent); + } + + const newResource = luParser.parseWithRef(sectionContent, this.Luresource); + if (!newResource) { return this.Luresource; } - const startLine = section ? section.StartLine : 0; - const newContent = this.replaceRangeContent(this.Luresource.Content, startLine, startLine - 1, sectionContent); + // add a NestedIntentSection may appears multiple sections - const result = luParser.parse(newContent); + // adjust original errors + const newLineRange = sectionContent.split(/\r?\n/).length; + const startLine = sectionIndex <= 0 ? 1 : this.Luresource.Sections[sectionIndex].Range.Start.Line; + this.adjustRangeForErrors(this.Luresource.Errors, newLineRange, startLine); - return result; + // adjust the insert errors of section + this.adjustRangeForErrors(newResource.Errors, startLine - 1); + this.Luresource.Errors.push(...newResource.Errors); + + this.Luresource.Content = this.replaceRangeContent(this.Luresource.Content, startLine - 1, startLine - 2, sectionContent); + this.adjustRangeForInsertSection(sectionIndex, newResource.Sections); + + luParser.extractSectionBody(this.Luresource.Sections, this.Luresource.Content); + if (this.Luresource.hashATNConfig) { + console.log('----------------------------'); + } + return this.Luresource; + } + + removeErrors(errors, startLine, endLine) { + if (errors && startLine >= 0 && endLine >= startLine) { + let index = -1; + + while ((index = errors.findIndex(u => + u.Range && ((u.Range.Start.Line >= startLine && u.Range.Start.Line <= endLine) + || (u.Range.End.Line >= startLine && u.Range.End.Line <= endLine)))) >= 0) { + this.Luresource.Errors.splice(index, 1); + } + } + } + + adjustRangeForErrors(errors, offset, startLine, endLine) { + if (errors) { + if (startLine === undefined && endLine === undefined) { + errors.forEach(u => { + this.adjustErrorRange(u, offset); + }); + } else if (startLine >= 0 && (endLine === undefined || endLine < startLine)) { + errors.forEach(u => { + if (u.Range.Start.Line >= startLine) { + this.adjustErrorRange(u, offset); + } + }); + } else if (startLine >= 0 && endLine >= startLine) { + errors.forEach(u => { + if (u.Range.Start.Line >= startLine && u.Range.End.Line <= endLine) { + this.adjustErrorRange(u, offset); + } + }); + } + } + } + + adjustErrorRange(error, offset) { + if (error && error.Range) { + error.Range.Start.Line += offset; + error.Range.End.Line += offset; + } + } + + adjustRangeForAddSection(newSections, offset) { + newSections.forEach(u => { + this.adjustSectionRange(u, offset); + }); + } + + adjustSectionRange(section, offset) { + if (section) { + if (section.SimpleIntentSections && section.SectionType === LUSectionTypes.NESTEDINTENTSECTION && section.SimpleIntentSections) { + section.SimpleIntentSections.forEach(k => { + k.Range.Start.Line += offset; + k.Range.End.Line += offset; + }); + } + + section.Range.Start.Line += offset; + section.Range.End.Line += offset; + } + } + + adjustRangeForDeleteSection(index, offset) { + for (let i = index; i < this.Luresource.Sections.length; i++) { + const section = this.Luresource.Sections[i]; + this.adjustSectionRange(section, -offset) + } + } + + adjustRangeForUpdateSection(oldIndex, newSections) { + const sectionsSize = newSections.length; + const oldStartLine = this.Luresource.Sections[oldIndex].Range.Start.Line; + const oldEndLine = this.Luresource.Sections[oldIndex].Range.End.Line; + const newStartLine = newSections[0].Range.Start.Line; + const newEndLine = newSections[newSections.length - 1].Range.End.Line; + + this.Luresource.Sections.splice(oldIndex, 1, ...newSections); + + // adjust updated sections' range + const updateOffset = oldStartLine - this.Luresource.Sections[oldIndex].Range.Start.Line; + for (let i = oldIndex; i < oldIndex + sectionsSize; i++) { + const section = this.Luresource.Sections[i]; + this.adjustSectionRange(section, updateOffset); + } + + // adjust remaining sections' range + const remainingOffset = (newEndLine - newStartLine) - (oldEndLine - oldStartLine); + for (let i = oldIndex + sectionsSize; i < this.Luresource.Sections.length; i++) { + const section = this.Luresource.Sections[i]; + this.adjustSectionRange(section, remainingOffset); + } + } + + adjustRangeForInsertSection(postIndex, newSections) { + const sectionsSize = newSections.length; + const insertOffset = postIndex <= 0 ? 0 : this.Luresource.Sections[postIndex].Range.Start.Line - 1; + const newStartLine = newSections[0].Range.Start.Line; + const newEndLine = newSections[newSections.length - 1].Range.End.Line; + + this.Luresource.Sections.splice(postIndex, 0, ...newSections); + + // adjust inserted sections' range + for (let i = postIndex; i < postIndex + sectionsSize; i++) { + const section = this.Luresource.Sections[i]; + this.adjustSectionRange(section, insertOffset); + } + + // adjust remaining sections' range + const remainingOffset = newEndLine - newStartLine + 1; + for (let i = postIndex + sectionsSize; i < this.Luresource.Sections.length; i++) { + const section = this.Luresource.Sections[i]; + this.adjustSectionRange(section, remainingOffset); + } } replaceRangeContent(originString, startLine, stopLine, replaceString) { const originList = originString.split(/\r?\n/); let destList = []; - if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine + 1 || originList.length <= stopLine) { + if (isNaN(startLine) || isNaN(stopLine) || startLine < 0 || startLine > stopLine + 1) { throw new Error("index out of range."); } diff --git a/packages/lu/src/parser/lufile/simpleIntentSection.js b/packages/lu/src/parser/lufile/simpleIntentSection.js index abe85af2d..a5088a8c7 100644 --- a/packages/lu/src/parser/lufile/simpleIntentSection.js +++ b/packages/lu/src/parser/lufile/simpleIntentSection.js @@ -6,14 +6,17 @@ const DiagnosticSeverity = require('./diagnostic').DiagnosticSeverity; const BuildDiagnostic = require('./diagnostic').BuildDiagnostic; const LUSectionTypes = require('./../utils/enums/lusectiontypes'); const NEWLINE = require('os').EOL; +const BaseSection = require('./baseSection'); +const Range = require('./diagnostic').Range; +const Position = require('./diagnostic').Position; -class SimpleIntentSection { +class SimpleIntentSection extends BaseSection { /** * * @param {SimpleIntentSectionContext} parseTree */ constructor(parseTree, content) { - this.ParseTree = parseTree; + super(); this.SectionType = LUSectionTypes.SIMPLEINTENTSECTION; this.UtteranceAndEntitiesMap = []; this.Entities = []; @@ -30,6 +33,9 @@ class SimpleIntentSection { this.Errors = this.Errors.concat(result.errors); this.Id = `${this.SectionType}_${this.Name}`; this.Body = this.ExtractBody(parseTree, content) + const startPosition = new Position(parseTree.start.line, parseTree.start.column); + const stopPosition = new Position(parseTree.stop.line, parseTree.stop.column + parseTree.stop.text.length); + this.Range = new Range(startPosition, stopPosition); } } @@ -61,7 +67,11 @@ class SimpleIntentSection { })) }; if (utteranceAndEntities !== undefined) { - utteranceAndEntities.context = normalIntentStr; + utteranceAndEntities.contextText = normalIntentStr.getText(); + const startPosition = new Position(normalIntentStr.start.line, normalIntentStr.start.column); + const stopPosition = new Position(normalIntentStr.stop.line, normalIntentStr.stop.column + normalIntentStr.stop.text.length); + utteranceAndEntities.range = new Range(startPosition, stopPosition); + utteranceAndEntitiesMap.push(utteranceAndEntities); utteranceAndEntities.errorMsgs.forEach(errorMsg => errors.push(BuildDiagnostic({ message: errorMsg, diff --git a/packages/lu/test/parser/lufile/sectionapi.test.js b/packages/lu/test/parser/lufile/sectionapi.test.js index 239fa89f7..0a452d30a 100644 --- a/packages/lu/test/parser/lufile/sectionapi.test.js +++ b/packages/lu/test/parser/lufile/sectionapi.test.js @@ -135,8 +135,8 @@ describe('Section CRUD tests for intent', () => { }); it('add nestedIntentSection test with enableSections flag set', () => { - let newFileConent = `> !# @enableSections = true${NEWLINE}${NEWLINE}# CheckTodo${NEWLINE}## CheckUnreadTodo${NEWLINE}- check my unread todo${NEWLINE}- show my unread todos${NEWLINE}${NEWLINE}@ simple todoTitle${NEWLINE}${NEWLINE}## CheckDeletedTodo${NEWLINE}- check my deleted todo${NEWLINE}- show my deleted todos${NEWLINE}${NEWLINE}@ simple todoSubject`; - + let newFileConent = `# CheckTodo${NEWLINE}## CheckUnreadTodo${NEWLINE}- check my unread todo${NEWLINE}- show my unread todos${NEWLINE}${NEWLINE}@ simple todoTitle${NEWLINE}${NEWLINE}## CheckDeletedTodo${NEWLINE}- check my deleted todo${NEWLINE}- show my deleted todos${NEWLINE}${NEWLINE}@ simple todoSubject`; + luresource = luparser.parse(luresource.Content + NEWLINE + '> !# @enableSections = true'); luresource = new SectionOperator(luresource).addSection(newFileConent); assert.equal(luresource.Errors.length, 0); @@ -247,7 +247,8 @@ describe('Section CRUD tests for entity', () => { it('update entity section test', () => { let newFileConent = -` - multiGrainWheat: +`@ list BreadEntity = + - multiGrainWheat: - multi grain - white: - white`; @@ -400,6 +401,6 @@ hello assert.equal(luresource.Errors.length, 0); assert.equal(luresource.Sections.length, 1); assert.equal(luresource.Sections[0].SectionType, LUSectionTypes.QNASECTION); - assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), insertFileContent + '\n'); + assert.equal(luresource.Sections[0].Body.replace(/\r\n/g,"\n"), insertFileContent); }); }); \ No newline at end of file