fix(data qual ods): added skips to tests that fail as an example

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,6 +6,7 @@ Go to bottom for pull request template to complete
 - everyone should get to be reviewed
 - everyone should get to review others
 - hierachyless
+- questions and answers equally useful, answering a question helps you concrete own understanding too through having to articulate it
 
 ## What it should be
 - Collaborative :) :)

databricks.yml

@@ -29,6 +29,7 @@ bundle:
   uuid: ba682f8a-5d14-4ae7-a770-d6a359c0e835
 
 include:
+  #   - 'variables/*.yml' if execute in order might be able to put vars in folders based off headings, may affect running in personal area before deploy?
   - resources/**/*.yml
   - resources/**/**/*.yml
 # ** double astrix doesnt seem to do sub directories

docs/TODO.md

@@ -1,3 +1,16 @@
+**try moving vars to const files etc and out of bundle**
+include:
+    - 'variables/*.yml' if execute in order might be able to put vars in folders based off headings, may affect running in personal area before deploy?
+
+
+- seperating dabs
+  - can do bronze, silver, gold etc
+  - means modular deployment
+  - cicd would need to detect changes in folders in order to know which dab to deploy
+  - wheels again may be needed for code between dabs but maybe not
+  - cluster dealing with smaller sizes
+  - how to test
+
 - are requirments txt still needed is it due to python version and setup in git pipeline that i am not just using toml?
 - i have 2 dlt pipeline version check which is right its in silver resources
 - check deploys

resources/jobs/utils/full_data_refresh.yml

@@ -7,6 +7,7 @@ resources:
         - task_key: run_bronze_pipeline
           pipeline_task:
             pipeline_id: ${resources.pipelines.pipeline_ods_ingestion.id}
+
         # Exclude streaming if they are triggered whilst active an error occurs
         # - task_key: run_silver_pipeline
         #   depends_on:

resources/test/integration_test_job.yml

@@ -13,12 +13,13 @@ resources:
         #     
         ####################
 
-        - task_key: medallion_refresh
+        - task_key: medallion_refresher
           run_job_task:
             job_id: ${resources.jobs.medallion_refresh.id}
+          timeout_seconds: 1800  # fail after 30 min instead of hanging
 
         - task_key: run_all_tests
-          depends_on: [medallion_refresh] # For data quality tests we need to run all the pipelines to see the new result from them 
+          depends_on: [medallion_refresher] # For data quality tests we need to run all the pipelines to see the new result from them 
           # serverless instead # job_cluster_key: test_cluster
           notebook_task:
             # DAB knows to look relative to the YAML file location

tests/data-quality-tests/bronze/ods/test_contact_details.py

@@ -59,7 +59,7 @@ def test_data_loaded_within_24_hours(self, contact_details_table):
 
 class TestContactDetailsTelephoneFormat:
     """Validate telephone number formats"""
-    @pytest.mark.staging_skip(reason="Just to see job is passing the value")
+    @pytest.mark.dev_skip(reason="Skip because we dont want to resolve the fail its an example, Uuse this to see a fail in action")
     def test_tel_is_numeric_when_not_null(self, contact_details_table):
         """Tel column should only contain digits when populated"""
         invalid_tel = contact_details_table.filter(
@@ -74,7 +74,8 @@ def test_tel_is_numeric_when_not_null(self, contact_details_table):
             invalid_tel.select("OrganisationId", "Tel").show(10, truncate=False)
 
         assert invalid_count == 0, f"Found {invalid_count} telephone numbers with non-numeric characters"
-
+
+    @pytest.mark.dev_skip(reason="Skip because we dont want to resolve the fail its an example, Uuse this to see a fail in action")
     def test_tel_has_valid_uk_length(self, contact_details_table):
         """UK telephone numbers should be 10-11 digits"""
         invalid_length = contact_details_table.filter(
@@ -103,6 +104,7 @@ def test_fax_is_numeric_when_not_null(self, contact_details_table):
 class TestContactDetailsCompleteness:
     """Check data completeness"""
 
+    @pytest.mark.staging_skip(reason="Just to see job is passing the value")
     def test_organisation_id_not_null(self, contact_details_table):
         """OrganisationId is a required field"""
         null_count = contact_details_table.filter(