diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs new file mode 100644 index 0000000..5cec555 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs @@ -0,0 +1,38 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +public class DebugHandler : IIntentHandler +{ + public bool CanHandle(string intentName) + { + //This is the debug handler, we want it to be able to act on any intent. + return true; + } + + public void Handle(Intent intent, LuisMRResult result) + { + //Build up a string of information about the result we got from LUIS + StringBuilder sb = new StringBuilder(); + sb.AppendLine("Utterance: " + result.Context.PredictionText); + sb.AppendLine("Intent: " + intent.Name); + sb.AppendLine(" Score: " + intent.Score.ToString("P")); + sb.AppendLine("Entities: "); + foreach(string entityKey in result.PredictionResult.Entities.Keys) + { + sb.AppendLine(" Entity: " + entityKey); + foreach(Entity e in result.PredictionResult.Entities[entityKey]) + { + sb.AppendLine(" Value: " + e.Value); + sb.AppendLine(" Score: " + e.Score); + } + } + //Then write it to the console + Debug.Log(sb.ToString()); + } +} diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta new file mode 100644 index 0000000..ac4c512 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 7dd2616a162189f48853f744373866cd +timeCreated: 1518679031 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs new file mode 100644 index 0000000..2b873f2 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +public class EntityMetaData : MonoBehaviour +{ + public string EntityName; + public string EntityType; +} diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta new file mode 100644 index 0000000..7453692 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 191a39b4e52ca2042b4d6913e578660c +timeCreated: 1518674451 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs new file mode 100644 index 0000000..0ef0aa9 --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs @@ -0,0 +1,54 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using UnityEngine; + +/// +/// Resolves entities in our prediction with entities in the scene. This resolver finds scene gameobjects by name or type. +/// The name and type of a gameobject in the scene is defined with the monobehaviour 'EntityMetaData'. +/// +public class EntityMetaDataResolver : IEntityResolver +{ + public string[] validEntityNames = new string[] + { + "MR.InstanceName", + "MR.InstanceType" + }; + + /// + /// Find all the scene GameObjects that have names matching the Entity value + /// + /// + public void Resolve(LuisMRResult result) + { + //Collect any entities that match the entity names we're looking for + var predictionEntities = result.PredictionResult.Entities.Where(x => validEntityNames.Contains(x.Key)).SelectMany(y => y.Value); + + if (predictionEntities.Count() < 1) + return; + + //Join the list of scene objects with prediction entities to get matches in the scene + IEnumerable matchedEntities = + from entity in predictionEntities + let entityName = entity.Value.ToLower() + from sceneEntity in GameObject.FindObjectsOfType() + where entityName.Equals(sceneEntity.EntityName.ToLower()) || entityName.Equals(sceneEntity.EntityType.ToLower()) + select new EntityMap() + { + Entity = entity, + GameObject = sceneEntity.gameObject, + Resolver = this + }; + + //Add all our found entities to the result's entity map, which maps LUIS entities with scene entities. + foreach (EntityMap entityMap in matchedEntities) + { + result.Map(entityMap); + } + } +} + diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta new file mode 100644 index 0000000..81b473b --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: f2db6c0340c9eb741aec1ca4fb248e57 +timeCreated: 1518676170 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs new file mode 100644 index 0000000..9ec449c --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs @@ -0,0 +1,53 @@ +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; +using System; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.Events; + +public class GenericEntityIntentHandler : MonoBehaviour, IIntentHandler +{ + [Serializable] + public struct IntentAction + { + public string entityValueTrigger; + public List actions; + } + + public string intentName; + public string entityTypeName; + public IntentAction[] handledIntents; + + Dictionary> activateValues = new Dictionary>(); + + void Start() + { + foreach (IntentAction intentAction in handledIntents) + { + if(!activateValues.ContainsKey(intentAction.entityValueTrigger)) + activateValues.Add(intentAction.entityValueTrigger, intentAction.actions); + else + activateValues[intentAction.entityValueTrigger].AddRange(intentAction.actions); + } + } + + public bool CanHandle(string intentName) + { + return intentName.Equals(intentName); + } + + public void Handle(Intent intent, LuisMRResult result) + { + if (!result.PredictionResult.Entities.ContainsKey(entityTypeName)) + return; + + foreach (Entity entity in result.PredictionResult.Entities[entityTypeName]) + { + if (activateValues.ContainsKey(entity.Value)) + { + foreach(UnityEvent uEvent in activateValues[entity.Value]) + uEvent.Invoke(); + } + } + } +} \ No newline at end of file diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta new file mode 100644 index 0000000..398a5ce --- /dev/null +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs.meta @@ -0,0 +1,13 @@ +fileFormatVersion: 2 +guid: 483a07b36eeff064c81164cce49f3279 +timeCreated: 1518664928 +licenseType: Pro +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs index 2565389..665ae21 100644 --- a/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs +++ b/Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs @@ -1,4 +1,5 @@ -using Microsoft.MR.LUIS; +using Microsoft.Cognitive.LUIS; +using Microsoft.MR.LUIS; using System.Collections; using System.Collections.Generic; using UnityEngine; @@ -11,7 +12,10 @@ public class LUISTester : MonoBehaviour // Use this for initialization async void Start() { + luisManager.EntityResolvers.Add(new EntityMetaDataResolver()); + luisManager.IntentHandlers.Add(new DebugHandler()); + var result = await luisManager.PredictAndHandle(testUtterence); - Debug.Log($"Utterence '{testUtterence}' confidence: {result.PredictionResult.TopScoringIntent.Score} was handled: {result.Handled}."); + } } diff --git a/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs b/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs index 7163410..d827a1b 100644 --- a/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs +++ b/Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs @@ -60,6 +60,27 @@ public List GetAllEntities() return list; } + /// + /// Maps the specified entity to the specified game object. + /// + /// + /// The set of context entity data to map. + /// + + public void Map(EntityMap contextEntityMap) + { + // First, make sure the entity table is created + List entityMapList; + if (!entities.TryGetValue(contextEntityMap.Entity.Name, out entityMapList)) + { + entities[contextEntityMap.Entity.Name] = new List() { contextEntityMap }; + } + else + { + entityMapList.Add(contextEntityMap); + } + } + /// /// Maps the specified entity to the specified game object. /// @@ -81,16 +102,7 @@ public void Map(Entity entity, GameObject gameObject, IEntityResolver resolver) // Create the map entry EntityMap map = new EntityMap { Entity = entity, GameObject = gameObject, Resolver = resolver }; - // First, make sure the entity table is created - List entityMapList; - if (!entities.TryGetValue(entity.Name, out entityMapList)) - { - entities[entity.Name] = new List() { map }; - } - else - { - entityMapList.Add(map); - } + Map(map); } #endregion // Public Methods diff --git a/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs b/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs index 4f2ff44..19ccf52 100644 --- a/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs +++ b/Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs @@ -323,6 +323,22 @@ public List ContextProviders } } + /// + /// Gets or sets the list of classes that can resolve LUIS Entity to GameObject relationships. + /// + public List EntityResolvers + { + get + { + return entityResolvers; + } + set + { + if (value == null) throw new ArgumentNullException(nameof(value)); + this.entityResolvers = value; + } + } + /// /// Gets or sets the list of handlers for LUIS intents. ///