DonJayamanne · DonJayamanne · Sep 23, 2020 · Sep 23, 2020 · Sep 23, 2020 · Sep 23, 2020
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -15,3 +15,10 @@ For #
 -   [ ] [Test plan](https://github.com/Microsoft/vscode-python/blob/main/.github/test_plan.md) is updated as appropriate.
 -   [ ] [`package-lock.json`](https://github.com/Microsoft/vscode-python/blob/main/package-lock.json) has been regenerated by running `npm install` (if dependencies have changed).
 -   [ ] The wiki is updated with any design decisions/details.
+
+
+**Optional tests to run**:
+Select one or more of the following tests to run as part of the CI for the current PR. If multiple tests are selected and one of them fails, then the rest will not run.
+* [ ] Run single-workspace tests (with VS Code, without Python extension & without Jupyter)
+* [ ] Run functional-with-jupyter tests (mocked VS Code, without Python extension & with Jupyter)
+* [ ] Run integration tests (with VS Code, with Python extension & with Jupyter)
diff --git a/.github/actions/build-vsix/action.yml b/.github/actions/build-vsix/action.yml
@@ -9,17 +9,17 @@ outputs:
 runs:
   using: 'composite'
   steps:
-    # For faster/better builds of sdists.
-    - run: python -m pip install wheel
-      shell: bash
+    # # For faster/better builds of sdists.
+    # - run: python -m pip install wheel
+    #   shell: bash
 
-    - run: python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
-      shell: bash
+    # - run: python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+    #   shell: bash
 
-    - run: |
-        python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt
-        python ./pythonFiles/install_debugpy.py
-      shell: bash
+    # - run: |
+    #     python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt
+    #     python ./pythonFiles/install_debugpy.py
+    #   shell: bash
 
     - run: npm ci --prefer-offline
       shell: bash

diff --git a/.github/workflows/flaky.yml b/.github/workflows/flaky.yml
@@ -0,0 +1,163 @@
+name: Flaky Tests
+
+on:
+  push:
+    branches:
+      - main
+      - ci
+  workflow_dispatch:
+
+env:
+  NODE_VERSION: 12.15.0
+  PYTHON_VERSION: 3.8
+  CACHE_NPM_DEPS: cache-npm
+  CACHE_OUT_DIRECTORY: cache-out-directory
+  CACHE_PIP_DEPS: cache-pip
+  CI_PYTHON_PATH: python
+  TEST_RESULTS_DIRECTORY: .
+
+jobs:
+  tests:
+    name: Tests (with Python)
+    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
+    runs-on: ${{ matrix.os }}
+    if: github.repository == 'microsoft/vscode-jupyter'
+    strategy:
+      fail-fast: false
+      matrix:
+        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
+        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
+        os: [ubuntu-latest, windows-latest]
+        python: [3.6, 3.7, 3.8]
+        test-suite: [python-unit, single-workspace, functional, functional-with-jupyter]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Use Python ${{matrix.python}}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{matrix.python}}
+
+      - name: Upgrade pip
+        run: python -m pip install -U pip
+
+      - name: Use Node ${{env.NODE_VERSION}}
+        uses: actions/setup-node@v2.1.1
+        with:
+          node-version: ${{env.NODE_VERSION}}
+
+      # Start caching
+
+      # Cache Python Dependencies.
+      # Caching (https://github.com/actions/cache/blob/main/examples.md#python---pip
+      - name: Cache pip on linux
+        uses: actions/cache@v2
+        # if: startsWith(runner.os, 'Linux')
+        if: startsWith(matrix.test-suite, 'functional') && matrix.os == 'ubuntu-latest'
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      - name: Cache pip on mac
+        uses: actions/cache@v2
+        if: startsWith(matrix.test-suite, 'functional')  && matrix.os == 'macos-latest'
+        with:
+          path: ~/Library/Caches/pip
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      - name: Cache pip on windows
+        uses: actions/cache@v2
+        # if: startsWith(runner.os, 'Windows')
+        if: startsWith(matrix.test-suite, 'functional')  && matrix.os == 'windows-latest'
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-
+
+      # Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)
+      - name: Cache npm on linux/mac
+        uses: actions/cache@v2
+        if: matrix.os != 'windows-latest'
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Get npm cache directory
+        if: matrix.os == 'windows-latest'
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$(npm config get cache)"
+      - name: Cache npm on windows
+        uses: actions/cache@v2
+        if: matrix.os == 'windows-latest'
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Cache compiled TS files
+        # Use an id for this step so that its cache-hit output can be accessed and checked in the next step.
+        id: out-cache
+        uses: actions/cache@v2
+        with:
+          path: ./out
+          key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}
+
+      # For faster/better builds of sdists.
+      - run: python -m pip install wheel
+        if: startsWith(matrix.test-suite, 'functional')
+        shell: bash
+
+      # debugpy  & Jedi LS is not shipped, only installed for local tests.
+      # In production, we get debugpy & LS from python extension.
+      - name: Install functional test requirements
+        #   python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+        run: |
+          python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt
+          python ./pythonFiles/install_debugpy.py
+          python -m pip install numpy
+          python -m pip install --upgrade -r build/test-requirements.txt
+          python -m pip install --upgrade -r ./build/ipython-test-requirements.txt
+          python -m pip install --upgrade -r ./build/conda-functional-requirements.txt
+
+      - name: Install dependencies (npm ci)
+        run: npm ci --prefer-offline
+
+      - name: Compile if not cached
+        run: npx gulp prePublishNonBundle
+        if: steps.out-cache.outputs.cache-hit == false
+
+      # Run the Python and IPython tests in our codebase.
+      - name: Run Python and IPython unit tests
+        run: |
+          python pythonFiles/tests/run_all.py
+          python -m IPython pythonFiles/tests/run_all.py
+        if: matrix.test-suite == 'python-unit'
+
+      - name: Run single-workspace tests
+        env:
+          CI_PYTHON_VERSION: ${{matrix.python}}
+        uses: GabrielBB/xvfb-action@v1.4
+        with:
+          run: npm run testSingleWorkspace
+        if: matrix.test-suite == 'single-workspace'
+
+      - name: Run functional tests
+        run: npm run test:functional
+        if: matrix.test-suite == 'functional'
+
+      - name: Run functional tests with Jupyter
+        run: npm run test:functional
+        env:
+          VSCODE_PYTHON_ROLLING: 1
+          VSC_PYTHON_FORCE_LOGGING: 1
+        if: matrix.test-suite == 'functional-with-jupyter'