From 4082126153de6e5c6dd077d1b9dc986761deb6df Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:24:02 +0000 Subject: [PATCH 1/8] Add pyspark to development requirements --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 55b033e..8fa08a5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ -pytest \ No newline at end of file +pytest +pyspark From e41f649dda12c2b7656976efb5211f4529a69b01 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:28:16 +0000 Subject: [PATCH 2/8] Rename pytest step and specify unit tests directory --- .github/workflows/manual-trigger-test-poc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-trigger-test-poc.yml b/.github/workflows/manual-trigger-test-poc.yml index b02d834..c2b5a47 100644 --- a/.github/workflows/manual-trigger-test-poc.yml +++ b/.github/workflows/manual-trigger-test-poc.yml @@ -24,6 +24,6 @@ jobs: pip install -r requirements-dev.txt pip install -e . - - name: Run pytest (exclude Databricks tests) + - name: Run pytest againt unit tests (exclude Databricks tests) run: | - pytest -m "not databricks" -v \ No newline at end of file + pytest tests/unit-tests -m "not databricks" -v From 72e29aadc88309afe4cf653ea34f45409d48c244 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:37:58 +0000 Subject: [PATCH 3/8] Remove session-scoped Spark fixture Removed the session-scoped Spark fixture to simplify test setup. --- .../transformations/test_date_transformations.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/unit-tests/transformations/test_date_transformations.py b/tests/unit-tests/transformations/test_date_transformations.py index 9888c50..e1b129b 100644 --- a/tests/unit-tests/transformations/test_date_transformations.py +++ b/tests/unit-tests/transformations/test_date_transformations.py @@ -25,13 +25,7 @@ # return session -@pytest.fixture(scope="session") -def spark(): - """Provide Spark session for all tests""" - session = SparkSession.getActiveSession() - if session is None: - raise RuntimeError("No active Spark session found. Running in Databricks?") - return session + @pytest.fixture(scope="function") @@ -226,4 +220,4 @@ def test_working_days_values_are_reasonable(spark, sample_dataframe_multiple_mon assert 19 <= working_days <= 23, \ f"Working days should be between 19-23, got {working_days} for {row['start_date']}" - print("✅ All working days values are reasonable") \ No newline at end of file + print("✅ All working days values are reasonable") From 33d9f2b28ed1e3e3fa98a6d643ae2d0fae2ed09b Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:55:35 +0000 Subject: [PATCH 4/8] Enhance CI workflow with linting and validation steps Updated CI workflow to include linting, unit tests, and Databricks validation. --- .github/workflows/ci.yml | 92 ++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fba5fa1..08b7f42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,56 +1,66 @@ -# We want to run CI processes that can run independent of databricks as branch rules so that we dont # deploy at cost code that we already should know needs changing -# such as linting, and unit test for python, maybe dab? verify -# we run these on all pull request because if there is a hot fix it may not have passed through -# staging for example -# qqqq check this is up to date name: CI - Pull Request Checks -# Run CI on all pull requests +# Run CI on all pull requests just incase of hot fixes on: pull_request: branches: - '**' # all branches + workflow_dispatch: jobs: - ci_checks: - name: "Linting, Unit Tests, DAB Verify" + lint: + name: "Linting" + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Commit lint placeholder + # TODO: Implement commitlint with config from TELBlazor reference + run: | + echo "WARNING: Commit lint step is currently a placeholder." + echo "Reference project: TELBlazor" + # exit 0 <-- Change to 0 if you don't want to block the rest of the pipe yet + + pytest: runs-on: ubuntu-latest steps: - # Checkout code - - name: Checkout repository + - name: Check out repository uses: actions/checkout@v4 - # Set up Python - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.x" - - # Install dependencies used for linting and unit tests - - name: Install dependencies - run: pip install -r requirements-dev.txt - - # Run python unit tests - - name: Run Unit Tests - run: pytest tests/unit - - # Run python lint - # qqqq on example used flake8 instead - # pyproject.toml will need configuring - - name: Run Linting - run: pylint src - - # qqqq to do run commit lint step and put in commit lint config - # see TELBlazor - - name: Commit lint - run: | - echo "Commit lint not implemented" - exit 1 - - # qqqq to do run version generation step and put in commit lint config - # see TELBlazor - - name: Version Generation Test Run - run: | - echo "Version test run not implemented" - exit 1 + python-version: "3.10" + + - name: Upgrade pip + run: python -m pip install --upgrade pip + + - name: Install project + test deps + run: | + pip install -r requirements-dev.txt + pip install -e . + - name: Run pytest againt unit tests (exclude Databricks tests) + run: | + pytest tests/unit-tests -m "not databricks" -v + + dab-validate: + name: "Early warning of dab issues" + runs-on: ubuntu-latest + # Just dev we are not deploying so dont need access to more important service principle + environment: dev + env: + DATABRICKS_HOST: ${{ vars.DBX_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Databricks CLI + uses: databricks/setup-cli@main + + - name: Validate Bundle + run: databricks bundle validate From f42db777477185e50eef661e300ebb2e2b425875 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:58:44 +0000 Subject: [PATCH 5/8] Add manual integration test workflow for dbx This workflow triggers dbx integration tests based on user input for deployment target. --- .github/workflows/manual-integration-test-poc.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/manual-integration-test-poc.yml diff --git a/.github/workflows/manual-integration-test-poc.yml b/.github/workflows/manual-integration-test-poc.yml new file mode 100644 index 0000000..0c4a3cd --- /dev/null +++ b/.github/workflows/manual-integration-test-poc.yml @@ -0,0 +1,14 @@ +# This yml is so we can tell dbx to run integration tests +name: Trying to trigger dbx integration tests in dbx environ + +on: + workflow_dispatch: + inputs: + deploy_target: + description: 'Which DAB target to deploy to?' + required: true + default: 'dev' + type: choice + options: + - dev + - staging From bc4c519555ccb7bcb500d4497acaa9a198e3b5c3 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:01:30 +0000 Subject: [PATCH 6/8] Update CI environment to staging Changed environment from 'dev' to 'staging' in CI workflow. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08b7f42..28413c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,8 @@ jobs: name: "Early warning of dab issues" runs-on: ubuntu-latest # Just dev we are not deploying so dont need access to more important service principle - environment: dev + # !!!! TODO !!!!! Should be dev but git needs the service principle putting into its environment + environment: staging env: DATABRICKS_HOST: ${{ vars.DBX_HOST }} DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }} From 1fd3a21caa623e02886f79ca5566e7746ba2ed81 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:05:43 +0000 Subject: [PATCH 7/8] Update ci.yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28413c8..6765e68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,4 +64,5 @@ jobs: uses: databricks/setup-cli@main - name: Validate Bundle - run: databricks bundle validate + # Am i forced to have a target its after databrickcfg but shouldnt be + run: databricks bundle validate -t staging From 85740b4ff3d4f10049974ef380591a79073a6eb0 Mon Sep 17 00:00:00 2001 From: Phil <165780796+Phil-NHS@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:41:49 +0000 Subject: [PATCH 8/8] Add GitHub Actions workflow for staging integration tests --- .github/workflows/main.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..f18fdac --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,34 @@ +name: Just a POC Manual Integration Tests - Staging + +on: + workflow_dispatch: + +jobs: + integration_tests_staging: + name: "Deploy & Test in Staging" + runs-on: ubuntu-latest + # Using the staging environment for Service Principal secrets + environment: staging + + env: + # Assume the DAB bundle is already initialized in the repo but this step would actually come after dab deployment unless it can be run against the file directory + # Using Staging Service Principal credentials from Git Secrets/Vars + # using staging here because i havent set up env vars in git for dev yet + DATABRICKS_HOST: ${{ vars.DBX_HOST }} + DATABRICKS_CLIENT_ID: ${{ vars.DBX_SP_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.DBX_SP_SECRET }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Databricks CLI + uses: databricks/setup-cli@main + + # this step would come after dab deploy? or would run first dev dab? + # i dont want a fail really and the code to be sat there known to be bad so running locally via a hook could be good pre-push? + # The notebook will actually run both as set up atm but maybe its ok for now + - name: Run Integration Tests + run: | + echo "running the integration yml test job integration_test_job.yml" + databricks bundle run -t staging run_integration_tests