From 4ac259f0263e3e20a14eee0254677391ab3f7aff Mon Sep 17 00:00:00 2001 From: Michael House Date: Thu, 15 Feb 2018 09:52:03 -0800 Subject: [PATCH] Debug handler, for displaying debug information about all predicted intents Entity Meta Data resolver and monobehavior, for tagging scene entities to be identified by name or type Generic Entity Intent Handler, for adding UnityEvents to trigger when a gameobject recieves a specific value, in a specific entity type, in a specific intent --- .../LUIS/Scripts/DebugHandler.cs | 38 +++++++++++++ .../LUIS/Scripts/DebugHandler.cs.meta | 13 +++++ .../LUIS/Scripts/EntityMetaData.cs | 12 +++++ .../LUIS/Scripts/EntityMetaData.cs.meta | 13 +++++ .../LUIS/Scripts/EntityMetaDataResolver.cs | 54 +++++++++++++++++++ .../Scripts/EntityMetaDataResolver.cs.meta | 13 +++++ .../Scripts/GenericEntityIntentHandler.cs | 53 ++++++++++++++++++ .../GenericEntityIntentHandler.cs.meta | 13 +++++ .../LUIS/Scripts/LUISTester.cs | 8 ++- .../LUIS/Scripts/LuisMRResult.cs | 32 +++++++---- .../LUIS/Scripts/LuisManager.cs | 16 ++++++ 11 files changed, 253 insertions(+), 12 deletions(-) create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs create mode 100644 Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs new file mode 100644 index 0000000..5cec555 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs @@ -0,0 +1,38 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +public class DebugHandler : IIntentHandler +{ + public bool CanHandle(string intentName) + { + //This is the debug handler, we want it to be able to act on any intent. + return true; + } + + public void Handle(Intent intent, LuisMRResult result) + { + //Build up a string of information about the result we got from LUIS + StringBuilder sb = new StringBuilder(); + sb.AppendLine("Utterance: " + result.Context.PredictionText); + sb.AppendLine("Intent: " + intent.Name); + sb.AppendLine(" Score: " + intent.Score.ToString("P")); + sb.AppendLine("Entities: "); + foreach(string entityKey in result.PredictionResult.Entities.Keys) + { + sb.AppendLine(" Entity: " + entityKey); + foreach(Entity e in result.PredictionResult.Entities[entityKey]) + { + sb.AppendLine(" Value: " + e.Value); + sb.AppendLine(" Score: " + e.Score); + } + } + //Then write it to the console + Debug.Log(sb.ToString()); + } +} diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta new file mode 100644 index 0000000..ac4c512 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 7dd2616a162189f48853f744373866cd +timeCreated: 1518679031 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs new file mode 100644 index 0000000..2b873f2 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +public class EntityMetaData : MonoBehaviour +{ + public string EntityName; + public string EntityType; +} diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta new file mode 100644 index 0000000..7453692 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 191a39b4e52ca2042b4d6913e578660c +timeCreated: 1518674451 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs new file mode 100644 index 0000000..0ef0aa9 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs @@ -0,0 +1,54 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +/// +/// Resolves entities in our prediction with entities in the scene. This resolver finds scene gameobjects by name or type. +/// The name and type of a gameobject in the scene is defined with the monobehaviour 'EntityMetaData'. +/// +public class EntityMetaDataResolver : IEntityResolver +{ + public string[] validEntityNames = new string[] + { + "MR.InstanceName", + "MR.InstanceType" + }; + + /// + /// Find all the scene GameObjects that have names matching the Entity value + /// + /// + public void Resolve(LuisMRResult result) + { + //Collect any entities that match the entity names we're looking for + var predictionEntities = result.PredictionResult.Entities.Where(x => validEntityNames.Contains(x.Key)).SelectMany(y => y.Value); + + if (predictionEntities.Count() < 1) + return; + + //Join the list of scene objects with prediction entities to get matches in the scene + IEnumerable matchedEntities = + from entity in predictionEntities + let entityName = entity.Value.ToLower() + from sceneEntity in GameObject.FindObjectsOfType() + where entityName.Equals(sceneEntity.EntityName.ToLower()) || entityName.Equals(sceneEntity.EntityType.ToLower()) + select new EntityMap() + { + Entity = entity, + GameObject = sceneEntity.gameObject, + Resolver = this + }; + + //Add all our found entities to the result's entity map, which maps LUIS entities with scene entities. + foreach (EntityMap entityMap in matchedEntities) + { + result.Map(entityMap); + } + } +} + diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta new file mode 100644 index 0000000..81b473b --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: f2db6c0340c9eb741aec1ca4fb248e57 +timeCreated: 1518676170 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs new file mode 100644 index 0000000..9ec449c --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs @@ -0,0 +1,53 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.Events; + +public class GenericEntityIntentHandler : MonoBehaviour, IIntentHandler +{ + [Serializable] + public struct IntentAction + { + public string entityValueTrigger; + public List actions; + } + + public string intentName; + public string entityTypeName; + public IntentAction[] handledIntents; + + Dictionary> activateValues = new Dictionary>(); + + void Start() + { + foreach (IntentAction intentAction in handledIntents) + { + if(!activateValues.ContainsKey(intentAction.entityValueTrigger)) + activateValues.Add(intentAction.entityValueTrigger, intentAction.actions); + else + activateValues[intentAction.entityValueTrigger].AddRange(intentAction.actions); + } + } + + public bool CanHandle(string intentName) + { + return intentName.Equals(intentName); + } + + public void Handle(Intent intent, LuisMRResult result) + { + if (!result.PredictionResult.Entities.ContainsKey(entityTypeName)) + return; + + foreach (Entity entity in result.PredictionResult.Entities[entityTypeName]) + { + if (activateValues.ContainsKey(entity.Value)) + { + foreach(UnityEvent uEvent in activateValues[entity.Value]) + uEvent.Invoke(); + } + } + } +} \ No newline at end of file diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta new file mode 100644 index 0000000..398a5ce --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 483a07b36eeff064c81164cce49f3279 +timeCreated: 1518664928 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs index 2565389..665ae21 100644 --- a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs @@ -1,4 +1,5 @@ -using Microsoft.MR.LUIS; +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -11,7 +12,10 @@ public class LUISTester : MonoBehaviour // Use this for initialization async void Start() { + luisManager.EntityResolvers.Add(new EntityMetaDataResolver()); + luisManager.IntentHandlers.Add(new DebugHandler()); + var result = await luisManager.PredictAndHandle(testUtterence); - Debug.Log($"Utterence '{testUtterence}' confidence: {result.PredictionResult.TopScoringIntent.Score} was handled: {result.Handled}."); + } } diff --git a/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs b/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs index 7163410..d827a1b 100644 --- a/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs +++ b/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs @@ -60,6 +60,27 @@ public List GetAllEntities() return list; } + /// + /// Maps the specified entity to the specified game object. + /// + /// + /// The set of context entity data to map. + /// + + public void Map(EntityMap contextEntityMap) + { + // First, make sure the entity table is created + List entityMapList; + if (!entities.TryGetValue(contextEntityMap.Entity.Name, out entityMapList)) + { + entities[contextEntityMap.Entity.Name] = new List() { contextEntityMap }; + } + else + { + entityMapList.Add(contextEntityMap); + } + } + /// /// Maps the specified entity to the specified game object. /// @@ -81,16 +102,7 @@ public void Map(Entity entity, GameObject gameObject, IEntityResolver resolver) // Create the map entry EntityMap map = new EntityMap { Entity = entity, GameObject = gameObject, Resolver = resolver }; - // First, make sure the entity table is created - List entityMapList; - if (!entities.TryGetValue(entity.Name, out entityMapList)) - { - entities[entity.Name] = new List() { map }; - } - else - { - entityMapList.Add(map); - } + Map(map); } #endregion // Public Methods diff --git a/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs b/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs index 4f2ff44..19ccf52 100644 --- a/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs +++ b/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs @@ -323,6 +323,22 @@ public List ContextProviders } } + /// + /// Gets or sets the list of classes that can resolve LUIS Entity to GameObject relationships. + /// + public List EntityResolvers + { + get + { + return entityResolvers; + } + set + { + if (value == null) throw new ArgumentNullException(nameof(value)); + this.entityResolvers = value; + } + } + /// /// Gets or sets the list of handlers for LUIS intents. ///