Mix-tags-and-policy

docs/Cost Visibility.md

@@ -4,6 +4,7 @@
 Todo: how to assign tagging
 Todo: add reminder in pull request template
 For every notebook, a tag or query? to identify it???  
+Drop down databricks instances, settings, to usage is how to connect the policyies to azure
 
 ## Approaches
 Todo: table, dashboard, budget tagging, and where triggered, when to pick the tag, what the tage means
@@ -19,6 +20,13 @@ Todo: link dashboards etc encourage/invite looking at, improvements
 
 # What you dont need to know
 
+## Azure
+- Budgets minimum gradularity is 1 month
+
+## Budget calculation limitations
+[Microsoft link serverless budgets](https://learn.microsoft.com/en-us/azure/databricks/admin/account-settings/budgets#known-limitations)
+> Budgets do not factor in any billing credits or negotiated discounts your account might have. The spent amount is calculated by multiplying usage by the SKU list price.
+
 ## Setup Budget Policys
 
 00_ because defaultest to highest alphabetically. (We should change this to 5 to give head room)

resources/jobs/utils/full_data_refresh.yml

@@ -16,3 +16,13 @@ resources:
         #     - task_key: run_bronze_pipeline
         #   pipeline_task:
         #     pipeline_id: ${resources.pipelines.pipeline_isreporter_live_dlt.id}
+
+        # qqqq want to be set by job calling it
+      #usage_policy_id: ${var.env_serverless_usage_policies.env_lh_live_id}
+      # Tags set by the budget policy, though doesnt seem to provide tags for the ui qqqq check - it does turn up in workflow ui
+      tags:
+        env: "Med-shouldBeOverWrittenWithPolicy-${var.env_name}"
+        # qqqq want to be set by job calling it
+        # costCentre: "Med-shouldBeOverWrittenWithPolicy-${var.env_name}"
+        # test: "Med-shouldBeOverWrittenWithPolicy-all"
+        detail: "MEd-checkUIflowPickedUp"

resources/pipeline/silver/isreporter_dlt.yml

@@ -4,6 +4,13 @@ resources:
     pipeline_isreporter_live_dlt:
       name: isreporter_live_dlt
       usage_policy_id: ${var.env_serverless_usage_policies.env_lh_live_id}
+      # Tags set by the budget policy, though doesnt seem to provide tags for the ui qqqq check - it does turn up in workflow ui
+      tags:
+        env: "Report-shouldBeOverWrittenWithPolicy-${var.env_name}"
+        costCentre: "Report-shouldBeOverWrittenWithPolicy-${var.env_name}"
+        test: "Report-shouldBeOverWrittenWithPolicy-all"
+        detail: "checkUIflowPickedUp"
+      ###
       libraries:
         - glob:      # go up resources/pipelines/silver then down
             include: ../../../src/silver/isreporter_dlt.py
@@ -16,6 +23,11 @@ resources:
       catalog: ${var.catalog}
       target: ${var.schema_prefix}${var.layer_silver}_${var.domain_reporting}
       serverless: true
+
+
+
+
+
      # configuration:
         ############### Resource yml files for set of pipelines #################
         # If we do bronze, silver ... tranformation based layers with own yml files will define layer level vars here

resources/test/integration_test_job.yml

@@ -7,7 +7,12 @@ resources:
       name: "[${bundle.target}] Integration & Unit Test Runner"
 
       usage_policy_id: ${var.env_serverless_usage_policies.env_test_all_id}       
-
+      # Tags set by the budget policy, though doesnt seem to provide tags for the ui qqqq check
+      tags:
+        env: ${var.env_name}
+        costCentre: ""
+        test: "shouldbeoall"
+        detail: "checkUIflowPickedUp"
       tasks:
         ##############################################
         ####################