Skip to content

feat: Add LLM token usage tracking to Application Insights #586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Prachig-Microsoft wants to merge 23 commits into
base: dev
Choose a base branch
from
Open

feat: Add LLM token usage tracking to Application Insights #586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
28c6e96
feat: Add LLM token usage tracking to Application Insights
Prachig-Microsoft May 12, 2026
8bc8901
fix: Handle UsageDetails object and add missing dependencies
Prachig-Microsoft May 13, 2026
bcdcc25
feat: Add portable Azure Workbook template for token usage dashboard
Prachig-Microsoft May 13, 2026
902252a
fix: Update workbook template format for Advanced Editor import
Prachig-Microsoft May 14, 2026
bd3f1e0
fix: Replace tiles visualization with table to fix workbook import error
Prachig-Microsoft May 14, 2026
3852184
revert: Remove enableMonitoring from main.parameters.json
Prachig-Microsoft May 14, 2026
55eadb3
fix: Deduplicate KQL queries in workbook to prevent double-counting e…
Prachig-Microsoft May 14, 2026
6407673
feat: Update workbook with grid layout and add deploy-workbook.bicep
Prachig-Microsoft May 15, 2026
cdde8a3
feat: Change Token Usage by File Type to grid visualization
Prachig-Microsoft May 15, 2026
4e020ad
fix: derive overall token totals from per-agent events instead of bro…
Prachig-Microsoft May 15, 2026
c913352
feat: add processing time queries per pipeline step and per document
Prachig-Microsoft May 15, 2026
a6dcfbe
fix: rewrite processing time queries to use customEvents timestamps
Prachig-Microsoft May 15, 2026
1e81e75
chore: remove appi-pgcpfeatuw333 from workbook, keep only appi-cptoke…
Prachig-Microsoft May 15, 2026
5c9be24
Fix token usage workbook: dedup events, time range params, portable r…
Prachig-Microsoft May 15, 2026
d482a3c
Enable monitoring (Application Insights) for non-WAF deployment
Prachig-Microsoft May 18, 2026
85e9f49
Fix workbook cross-resource queries for separate RG deployment
Prachig-Microsoft May 18, 2026
f84a617
fix: workbook queries - dedup tokens, MapHandler-only doc count, inne…
Prachig-Microsoft May 20, 2026
6e04499
refactor: replace workbook-specific token tracking with generic telem…
Prachig-Microsoft Jun 1, 2026
5d85f65
fix: remove unused import and trailing blank line in test_token_usage…
Prachig-Microsoft Jun 1, 2026
a8fc83f
fix: add file_name and file_mime_type to token usage telemetry events
Prachig-Microsoft Jun 1, 2026
0d55e8c
feat: add deterministic event_id for deduplication in telemetry events
Prachig-Microsoft Jun 2, 2026
f4ee50e
Address PR #586 review comments
Prachig-Microsoft Jun 2, 2026
efebd81
fix: exclude telemetry module from Workflow coverage (tested in Conte…
Prachig-Microsoft Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 42 additions & 7 deletions infra/main.bicep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -881,7 +881,12 @@ module avmContainerApp 'br/public:avm/res/app/container-app:0.22.1' = {
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
enableTelemetry: enableTelemetry
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
Comment thread
Prachig-Microsoft marked this conversation as resolved.
managedIdentities: {
systemAssigned: true
userAssignedResourceIds: [
Expand Down Expand Up @@ -950,7 +955,12 @@ module avmContainerApp_API 'br/public:avm/res/app/container-app:0.22.1' = {
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
enableTelemetry: enableTelemetry
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down Expand Up @@ -1082,7 +1092,12 @@ module avmContainerApp_Web 'br/public:avm/res/app/container-app:0.22.1' = {
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
enableTelemetry: enableTelemetry
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down Expand Up @@ -1165,7 +1180,12 @@ module avmContainerApp_Workflow 'br/public:avm/res/app/container-app:0.22.1' = {
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
enableTelemetry: enableTelemetry
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down Expand Up @@ -1543,7 +1563,12 @@ module avmContainerApp_update 'br/public:avm/res/app/container-app:0.22.1' = {
enableTelemetry: enableTelemetry
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down Expand Up @@ -1625,7 +1650,12 @@ module avmContainerApp_API_update 'br/public:avm/res/app/container-app:0.22.1' =
enableTelemetry: enableTelemetry
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down Expand Up @@ -1761,7 +1791,12 @@ module avmContainerApp_Workflow_update 'br/public:avm/res/app/container-app:0.22
enableTelemetry: enableTelemetry
environmentResourceId: avmContainerAppEnv.outputs.resourceId
workloadProfileName: 'Consumption'
registries: null
registries: [
{
server: containerRegistryEndpoint
identity: avmContainerRegistryReader.outputs.resourceId
}
]
tags: tags
managedIdentities: {
systemAssigned: true
Expand Down
103 changes: 63 additions & 40 deletions infra/main.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"_generator": {
"name": "bicep",
"version": "0.42.1.51946",
"templateHash": "5885652317352749587"
"templateHash": "14320065740070986438"
},
"name": "Content Processing Solution Accelerator",
"description": "Bicep template to deploy the Content Processing Solution Accelerator with AVM compliance."
Expand Down Expand Up @@ -243,8 +243,6 @@
"bastionHostName": "[format('bas-{0}', variables('solutionSuffix'))]",
"jumpboxVmName": "[take(format('vm-{0}', variables('solutionSuffix')), 15)]",
"dataCollectionRulesResourceName": "[format('dcr-{0}', variables('solutionSuffix'))]",
"logAnalyticsWorkspaceResourceName": "[format('log-{0}', variables('solutionSuffix'))]",
"dcrLogAnalyticsDestinationName": "[format('la-{0}-destination', variables('logAnalyticsWorkspaceResourceName'))]",
"privateDnsZones": [
"privatelink.cognitiveservices.azure.com",
"privatelink.openai.azure.com",
Expand Down Expand Up @@ -14609,10 +14607,19 @@
{
"name": "SecurityAuditEvents",
"streams": [
"Microsoft-Event"
"Microsoft-WindowsEvent"
],
"eventLogName": "Security",
"eventTypes": [
{
"eventType": "Audit Success"
},
{
"eventType": "Audit Failure"
}
],
"xPathQueries": [
"Security!*[System[(band(Keywords,13510798882111488)) and (EventID != 4624)]]"
"Security!*[System[(EventID=4624 or EventID=4625)]]"
]
}
]
Expand All @@ -14621,7 +14628,7 @@
"logAnalytics": [
{
"workspaceResourceId": "[reference('logAnalyticsWorkspace').outputs.resourceId.value]",
"name": "[variables('dcrLogAnalyticsDestinationName')]"
"name": "[format('la-{0}', variables('dataCollectionRulesResourceName'))]"
}
]
},
Expand All @@ -14631,20 +14638,10 @@
"Microsoft-Perf"
],
"destinations": [
"[variables('dcrLogAnalyticsDestinationName')]"
"[format('la-{0}', variables('dataCollectionRulesResourceName'))]"
],
"transformKql": "source",
"outputStream": "Microsoft-Perf"
},
{
"streams": [
"Microsoft-Event"
],
"destinations": [
"[variables('dcrLogAnalyticsDestinationName')]"
],
"transformKql": "source",
"outputStream": "Microsoft-Event"
}
]
}
Expand Down Expand Up @@ -19275,7 +19272,7 @@
"mode": "Incremental",
"parameters": {
"name": {
"value": "[variables('logAnalyticsWorkspaceResourceName')]"
"value": "[format('log-{0}', variables('solutionSuffix'))]"
},
"location": {
"value": "[parameters('location')]"
Expand Down Expand Up @@ -28053,9 +28050,6 @@
"ipRules": []
}
},
"requireInfrastructureEncryption": {
"value": true
},
"supportsHttpsTrafficOnly": {
"value": true
},
Expand Down Expand Up @@ -36183,8 +36177,8 @@
"avmContainerApp_API",
"avmContainerApp_Workflow",
"avmManagedIdentity",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageBlob)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
"virtualNetwork"
]
},
Expand Down Expand Up @@ -42488,9 +42482,9 @@
"dependsOn": [
"avmAiServices",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').contentUnderstanding)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').openAI)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').aiServices)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').cognitiveServices)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').aiServices)]",
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').openAI)]",
"virtualNetwork"
]
},
Expand Down Expand Up @@ -44170,7 +44164,12 @@
"value": "[parameters('enableTelemetry')]"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"managedIdentities": {
"value": {
Expand Down Expand Up @@ -45801,7 +45800,12 @@
"value": "[parameters('enableTelemetry')]"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand Down Expand Up @@ -45916,9 +45920,6 @@
"ingressTransport": {
"value": "auto"
},
"ingressAllowInsecure": {
"value": false
},
"corsPolicy": {
"value": {
"allowedOrigins": [
Expand Down Expand Up @@ -47499,7 +47500,12 @@
"value": "[parameters('enableTelemetry')]"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand All @@ -47524,9 +47530,6 @@
"ingressTransport": {
"value": "auto"
},
"ingressAllowInsecure": {
"value": false
},
"scaleSettings": {
"value": {
"maxReplicas": "[if(parameters('enableScalability'), 3, 2)]",
Expand Down Expand Up @@ -49150,7 +49153,12 @@
"value": "[parameters('enableTelemetry')]"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand Down Expand Up @@ -50799,6 +50807,9 @@
"EnableMongo"
]
},
"enableAnalyticalStorage": {
"value": true
},
"defaultConsistencyLevel": {
"value": "Session"
},
Expand Down Expand Up @@ -61435,7 +61446,12 @@
"value": "Consumption"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand Down Expand Up @@ -63069,7 +63085,12 @@
"value": "Consumption"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand Down Expand Up @@ -63184,9 +63205,6 @@
"ingressTransport": {
"value": "auto"
},
"ingressAllowInsecure": {
"value": false
},
"corsPolicy": {
"value": {
"allowedOrigins": [
Expand Down Expand Up @@ -64769,7 +64787,12 @@
"value": "Consumption"
},
"registries": {
"value": null
"value": [
{
"server": "[parameters('containerRegistryEndpoint')]",
"identity": "[reference('avmContainerRegistryReader').outputs.resourceId.value]"
}
]
},
"tags": {
"value": "[parameters('tags')]"
Expand Down
3 changes: 3 additions & 0 deletions infra/main.parameters.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
},
"imageTag": {
"value": "${AZURE_ENV_IMAGETAG=latest_v2}"
},
"enableMonitoring": {
"value": false
}
Comment thread
Prachig-Microsoft marked this conversation as resolved.
Comment thread
Prachig-Microsoft marked this conversation as resolved.
}
}
1 change: 1 addition & 0 deletions src/ContentProcessor/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dependencies = [
"azure-ai-inference==1.0.0b9",
"azure-appconfiguration==1.8.0",
"azure-identity==1.26.0b1",
"azure-monitor-events-extension==0.1.0",
Comment thread
Prachig-Microsoft marked this conversation as resolved.
"azure-monitor-opentelemetry==1.8.7",
Comment thread
Prachig-Microsoft marked this conversation as resolved.
Comment thread
Prachig-Microsoft marked this conversation as resolved.
"azure-storage-blob==12.29.0b1",
"azure-storage-queue==12.16.0b1",
Expand Down
Loading
Loading