gluon-nlp/.github/workflows/unittests-gpu.yml at 47fccc381b7966a36fbbf1ef10e8f97adf0c4620 · dmlc/gluon-nlp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
name: continuous build - gpu

on: [push, pull_request_target]

defaults:
  run:
    shell: bash

jobs:
  unittest-gpu:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Install Linux dependencies
        run: sudo apt-get install libopenblas-dev

      - name: Setup python
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
          architecture: x64

      - name: Install Other Dependencies
        run: |
          python -m pip install --quiet --upgrade pip
          python -m pip install --quiet boto3

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Test project on AWS Batch(For push)
        if: ${{ github.event_name == 'push' }}
        run: |
          echo "Start submitting job"
          python ./tools/batch/submit-job.py --region us-east-1 \
                                             --job-type g4dn.4x \
                                             --name GluonNLP-GPU-Test-${{ github.ref }} \
                                             --source-ref ${{ github.ref }} \
                                             --work-dir . \
                                             --saved-output coverage.xml \
                                             --save-path coverage.xml \
                                             --remote https://github.com/${{ github.repository }} \
                                             --command "python3 -m pip install pytest-forked && python3 -m pytest --forked --cov=. --cov-config=./.coveragerc --cov-report=xml --durations=50 --device="gpu" --runslow ./tests/" \
                                             --wait | tee batch_job.log


      - name: Test project on AWS Batch(For pull request)
        if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
        run: |
          echo "Start submitting job"
          python ./tools/batch/submit-job.py --region us-east-1 \
                                             --job-type g4dn.4x \
                                             --name GluonNLP-GPU-Test-PR#${{ github.event.number }} \
                                             --source-ref ${{ github.event.pull_request.head.ref }} \
                                             --work-dir . \
                                             --saved-output coverage.xml \
                                             --save-path coverage.xml \
                                             --remote https://github.com/${{ github.event.pull_request.head.repo.full_name }} \
                                             --command "python3 -m pip install pytest-forked && python3 -m pytest --forked --cov=. --cov-config=./.coveragerc --cov-report=xml --durations=50 --device="gpu" --runslow ./tests/" \
                                             --wait | tee batch_job.log

      - name: Wait for job and copy files from AWS s3
        if: ${{ failure() || success() }}
        run: |
          head -100 batch_job.log | grep -oP -m 1 'jobId: \K(.*)' > jobid.log

          echo "Job ID is"
          cat jobid.log

          cat jobid.log | xargs -i python ./tools/batch/wait-job.py --job-id {}

          echo "Copy Codecov file"
          cat jobid.log | xargs -i aws s3api wait object-exists --bucket gluon-nlp-dev --key batch/{}/coverage.xml
          cat jobid.log | xargs -i aws s3 cp s3://gluon-nlp-dev/batch/{}/coverage.xml ./coverage.xml

      - name: Upload coverage to Codecov
        run: |
          curl -s https://codecov.io/bash -o codecov.sh
          if [ "$EVENT_NAME" == "push" ]; then \
              bash codecov.sh -f ./coverage.xml -n -F; \
          else \
              bash codecov.sh -f ./coverage.xml -n -F -C ${{ github.event.pull_request.head.sha }} -P ${{ github.event.pull_request.number }}; \
          fi
        env:
          EVENT_NAME: ${{ github.event_name }}

      - name: Upload Cloud Watch Log Stream
        if: ${{ failure() || success() }}
        uses: actions/upload-artifact@v2
        with:
          name: Test_Logs
          path: ./batch_job.log