DebugHandler, EntityMetaData+Resolver, GenericEntityIntentHandler

Assets/MixedRealityAzure-Examples/LUIS/Scripts/DebugHandler.cs

@@ -0,0 +1,38 @@
+using Microsoft.Cognitive.LUIS;
+using Microsoft.MR.LUIS;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using UnityEngine;
+
+public class DebugHandler : IIntentHandler
+{
+    public bool CanHandle(string intentName)
+    {
+        //This is the debug handler, we want it to be able to act on any intent.
+        return true;
+    }
+
+    public void Handle(Intent intent, LuisMRResult result)
+    {
+        //Build up a string of information about the result we got from LUIS
+        StringBuilder sb = new StringBuilder();
+        sb.AppendLine("Utterance: " + result.Context.PredictionText);
+        sb.AppendLine("Intent: " + intent.Name);
+        sb.AppendLine("  Score: " + intent.Score.ToString("P"));
+        sb.AppendLine("Entities: ");
+        foreach(string entityKey in result.PredictionResult.Entities.Keys)
+        {
+            sb.AppendLine("  Entity: " + entityKey);
+            foreach(Entity e in result.PredictionResult.Entities[entityKey])
+            {
+                sb.AppendLine("    Value:  " + e.Value);
+                sb.AppendLine("    Score:  " + e.Score);
+            }
+        }
+        //Then write it to the console
+        Debug.Log(sb.ToString());
+    }
+}

Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaData.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using UnityEngine;
+
+public class EntityMetaData : MonoBehaviour
+{
+    public string EntityName;
+    public string EntityType;
+}

Assets/MixedRealityAzure-Examples/LUIS/Scripts/EntityMetaDataResolver.cs

@@ -0,0 +1,54 @@
+using Microsoft.Cognitive.LUIS;
+using Microsoft.MR.LUIS;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using UnityEngine;
+
+/// <summary>
+/// Resolves entities in our prediction with entities in the scene. This resolver finds scene gameobjects by name or type.
+/// The name and type of a gameobject in the scene is defined with the monobehaviour 'EntityMetaData'.
+/// </summary>
+public class EntityMetaDataResolver : IEntityResolver
+{
+    public string[] validEntityNames = new string[]
+    {
+        "MR.InstanceName",
+        "MR.InstanceType"
+    };
+
+    /// <summary>
+    /// Find all the scene GameObjects that have names matching the Entity value
+    /// </summary>
+    /// <param name="result"></param>
+    public void Resolve(LuisMRResult result)
+    {
+        //Collect any entities that match the entity names we're looking for
+        var predictionEntities = result.PredictionResult.Entities.Where(x => validEntityNames.Contains(x.Key)).SelectMany(y => y.Value);
+
+        if (predictionEntities.Count() < 1)
+            return;
+
+        //Join the list of scene objects with prediction entities to get matches in the scene
+        IEnumerable<EntityMap> matchedEntities =
+            from entity in predictionEntities
+            let entityName = entity.Value.ToLower()
+            from sceneEntity in GameObject.FindObjectsOfType<EntityMetaData>()
+            where entityName.Equals(sceneEntity.EntityName.ToLower()) || entityName.Equals(sceneEntity.EntityType.ToLower())
+            select new EntityMap()
+            {
+                Entity = entity,
+                GameObject = sceneEntity.gameObject,
+                Resolver = this
+            };
+
+        //Add all our found entities to the result's entity map, which maps LUIS entities with scene entities.
+        foreach (EntityMap entityMap in matchedEntities)
+        {
+            result.Map(entityMap);
+        }
+    }
+}
+

Assets/MixedRealityAzure-Examples/LUIS/Scripts/GenericEntityIntentHandler.cs

@@ -0,0 +1,53 @@
+using Microsoft.Cognitive.LUIS;
+using Microsoft.MR.LUIS;
+using System;
+using System.Collections.Generic;
+using UnityEngine;
+using UnityEngine.Events;
+
+public class GenericEntityIntentHandler : MonoBehaviour, IIntentHandler
+{
+    [Serializable]
+    public struct IntentAction
+    {
+        public string entityValueTrigger;
+        public List<UnityEvent> actions;
+    }
+
+    public string intentName;
+    public string entityTypeName;
+    public IntentAction[] handledIntents;
+
+    Dictionary<string, List<UnityEvent>> activateValues = new Dictionary<string, List<UnityEvent>>();
+
+    void Start()
+    {
+        foreach (IntentAction intentAction in handledIntents)
+        {
+            if(!activateValues.ContainsKey(intentAction.entityValueTrigger))
+                activateValues.Add(intentAction.entityValueTrigger, intentAction.actions);
+            else
+                activateValues[intentAction.entityValueTrigger].AddRange(intentAction.actions);
+        }
+    }
+
+    public bool CanHandle(string intentName)
+    {
+        return intentName.Equals(intentName);
+    }
+
+    public void Handle(Intent intent, LuisMRResult result)
+    {
+        if (!result.PredictionResult.Entities.ContainsKey(entityTypeName))
+            return;
+
+        foreach (Entity entity in result.PredictionResult.Entities[entityTypeName])
+        {
+            if (activateValues.ContainsKey(entity.Value))
+            {
+                foreach(UnityEvent uEvent in activateValues[entity.Value])
+                    uEvent.Invoke();
+            }
+        }
+    }
+}

Assets/MixedRealityAzure-Examples/LUIS/Scripts/LUISTester.cs

@@ -1,4 +1,5 @@
-using Microsoft.MR.LUIS;
+using Microsoft.Cognitive.LUIS;
+using Microsoft.MR.LUIS;
 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
@@ -11,7 +12,10 @@ public class LUISTester : MonoBehaviour
     // Use this for initialization
     async void Start()
     {
+        luisManager.EntityResolvers.Add(new EntityMetaDataResolver());
+        luisManager.IntentHandlers.Add(new DebugHandler());
+
         var result = await luisManager.PredictAndHandle(testUtterence);
-        Debug.Log($"Utterence '{testUtterence}' confidence: {result.PredictionResult.TopScoringIntent.Score} was handled: {result.Handled}.");
+
     }
 }

Assets/MixedRealityAzure/LUIS/Scripts/LuisMRResult.cs

@@ -60,6 +60,27 @@ public List<EntityMap> GetAllEntities()
             return list;
         }
 
+        /// <summary>
+        /// Maps the specified entity to the specified game object.
+        /// </summary>
+        /// <param name="contextEntityMap">
+        /// The set of context entity data <see cref="ContextEntityMap"/> to map.
+        /// </param>
+
+        public void Map(EntityMap contextEntityMap)
+        {
+            // First, make sure the entity table is created
+            List<EntityMap> entityMapList;
+            if (!entities.TryGetValue(contextEntityMap.Entity.Name, out entityMapList))
+            {
+                entities[contextEntityMap.Entity.Name] = new List<EntityMap>() { contextEntityMap };
+            }
+            else
+            {
+                entityMapList.Add(contextEntityMap);
+            }
+        }
+
         /// <summary>
         /// Maps the specified entity to the specified game object.
         /// </summary>
@@ -81,16 +102,7 @@ public void Map(Entity entity, GameObject gameObject, IEntityResolver resolver)
             // Create the map entry
             EntityMap map = new EntityMap { Entity = entity, GameObject = gameObject, Resolver = resolver };
 
-            // First, make sure the entity table is created
-            List<EntityMap> entityMapList;
-            if (!entities.TryGetValue(entity.Name, out entityMapList))
-            {
-                entities[entity.Name] = new List<EntityMap>() { map };
-            }
-            else
-            {
-                entityMapList.Add(map);
-            }
+            Map(map);
         }
         #endregion // Public Methods
 

Assets/MixedRealityAzure/LUIS/Scripts/LuisManager.cs

@@ -323,6 +323,22 @@ public List<IContextProvider> ContextProviders
             }
         }
 
+        /// <summary>
+        /// Gets or sets the list of classes that can resolve LUIS Entity to GameObject relationships.
+        /// </summary>
+        public List<IEntityResolver> EntityResolvers
+        {
+            get
+            {
+                return entityResolvers;
+            }
+            set
+            {
+                if (value == null) throw new ArgumentNullException(nameof(value));
+                this.entityResolvers = value;
+            }
+        }
+
         /// <summary>
         /// Gets or sets the list of handlers for LUIS intents.
         /// </summary>