-
Notifications
You must be signed in to change notification settings - Fork 2
263 lines (240 loc) · 10.6 KB
/
perf-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
name: MIGraphX Performance Tests
on:
workflow_call:
inputs:
rocm_release:
type: string
description: ROCm release version
required: true
performance_reports_repo:
type: string
description: Result repository
required: true
benchmark_utils_repo:
type: string
description: Repository where benchmark utils are stored
required: true
organization:
type: string
description: Organization based on which location of files will be different
required: true
result_number:
type: string
description: Number of last results
required: true
model_timeout:
type: string
description: If model in performance test script passes this threshold, it will be skipped
required: true
flags:
type: string
description: -m for Max value; -s for Std dev; -r 'path' for Threshold file
required: true
performance_backup_repo:
type: string
description: Repository for backup
required: true
secrets:
gh_token:
description: 'Github Access Token'
required: true
mail_user:
description: 'Email username'
required: true
mail_pass:
description: 'Email password'
required: true
env:
UTILS_DIR: benchmark-utils
REPORTS_DIR: migraphx-reports
DOCKERBASE: rocm-migraphx:${{ inputs.rocm_release }}
MIOPENTUNE: miopen-dbs/rocm${{ inputs.rocm_release }}
MAIL_TO: dl.dl-migraphx-perfrun@amd.com
MAIL_CC: igor.mirosavljevic@htecgroup.com,djordje.petrovic@htecgroup.com,milos.miljanovic@htecgroup.com,pavle.jacovic@htecgroup.com
MAIL_FROM: GH Actions
MAIL_SUBJECT: Nightly Performance run
MAIL_BODY: Scheduled Performance test run on develop branch
PR_ID: ${{ github.event.number }}
BRANCH_NAME: ${{ github.base_ref }}
REPORTS_PATH: /usr/share/migraphx/${{ inputs.organization }}/reports
TEST_RESULTS_PATH: /usr/share/migraphx/${{ inputs.organization }}/test-results
MIGRAPHX_PATH: /usr/share/migraphx/${{ inputs.organization }}
PERFORMANCE_DIR: performance-backup
jobs:
performance_test:
name: MIGraphX Performance
if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip bot checks') }}
runs-on: self-hosted
outputs:
git_sha: ${{ steps.git_sha.outputs.git_sha }}
steps:
- name: Update Mailing list based on organization
if: ${{ inputs.organization == 'HTEC' }}
run: |
echo "MAIL_TO=$(echo "igor.mirosavljevic@htecgroup.com,djordje.petrovic@htecgroup.com,milos.miljanovic@htecgroup.com,pavle.jacovic@htecgroup.com")" >> $GITHUB_ENV
echo "MAIL_CC= " >> $GITHUB_ENV
- name: Update PR env
if: ${{ github.event_name == 'pull_request'}}
run: |
echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
echo "TEST_RESULTS_PATH=$(echo "$TEST_RESULTS_PATH-$PR_ID")" >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v3
- name: Checkout utils
uses: actions/checkout@v3
with:
repository: ${{ inputs.benchmark_utils_repo }}
path: ${{ env.UTILS_DIR }}
token: ${{ secrets.gh_token }}
- name: Get git SHA
id: git_sha
run: |
cd $GITHUB_WORKSPACE
SHA=$(git log | head -1 | awk '{print $2}' | cut -c 1-6)
echo "git_sha=$SHA" >> $GITHUB_OUTPUT
- name: Docker build
if: ${{ github.event.action != 'closed' }}
run: >
cd $GITHUB_WORKSPACE/${{ env.UTILS_DIR }} && docker build --no-cache
--build-arg BRANCH=${{ env.BRANCH_NAME }}
--build-arg DOCKERBASE=${{ env.DOCKERBASE }}
--build-arg MIOPENTUNE=${{ env.MIOPENTUNE }}
--build-arg benchmark_utils_repo=${{ inputs.benchmark_utils_repo }}
-t "migraphx-rocm:${{ inputs.rocm_release }}-${{ steps.git_sha.outputs.git_sha }}"
-f dockerfiles/Daily.Dockerfile .
- name: Run performance tests
if: ${{ github.event.action != 'closed' }}
run: >
docker run -e TZ=America/Chicago
-e TARGET=gpu
-e PYTHONPATH=/src/AMDMIGraphX/build/lib
--device=/dev/dri
--device=/dev/kfd
--network=host
--group-add=video
-v /usr/share/migraphx/saved-models:/saved-models:ro
-v /usr/share/migraphx/new-saved-models:/new-saved-models:ro
-v $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts:/migraphx/sh:ro
-v ${{ env.TEST_RESULTS_PATH }}:/data/test-results
--workdir /migraphx/sh
migraphx-rocm:${{ inputs.rocm_release }}-${{ steps.git_sha.outputs.git_sha }} /bin/bash performance_tests2.sh ${{ inputs.model_timeout }}
- name: Delete old images/containers
if: ${{ github.event_name == 'schedule' }}
run: |
docker container prune -f
if [[ $(docker images --filter=reference="migraphx-rocm:${{ inputs.rocm_release }}-*" | awk 'NR>1 {print $3}') ]]; then
docker rmi -f $(docker images --filter=reference="migraphx-rocm:${{ inputs.rocm_release }}-*" | awk 'NR>1 {print $3}')
fi
docker image prune -f
- name: Checkout report's repo
if: ${{ github.event_name == 'schedule' }}
uses: actions/checkout@v3
with:
repository: ${{ inputs.performance_reports_repo }}
path: ${{ env.REPORTS_DIR }}
token: ${{ secrets.gh_token }}
- name: Execute report script
if: ${{ github.event_name == 'schedule' }}
run: |
python3 $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts/report.py \
-t '${{ env.TEST_RESULTS_PATH }}' \
-r '${{ env.REPORTS_PATH }}'
python3 $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts/history.py -n \
-t '${{ env.TEST_RESULTS_PATH }}' \
-r '${{ env.REPORTS_PATH }}'
cp -r ${{ env.REPORTS_PATH }}/results-* $GITHUB_WORKSPACE/${{ env.REPORTS_DIR }}/
cd $GITHUB_WORKSPACE/${{ env.REPORTS_DIR }}/
git add .
git config --local user.email github-actions@github.com
git config --local user.name github-actions
git commit -m "Push report" -a
git push
- name: Execute comment script
id: auto_comment
if: ${{ github.event_name != 'schedule' && github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
if [ ${{ inputs.flags }} == "-r" ]; then
flagoptions="${{ inputs.flags }} $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts/"
else
flagoptions="${{ inputs.flags }}"
fi
python3 $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts/comment.py -t ${TEST_RESULTS_PATH//-$PR_ID/} -n ${{ inputs.result_number }} $flagoptions -p
- name: Create a comment on PR
if: ${{ github.event_name != 'schedule' && github.event_name == 'pull_request' && github.event.action != 'closed'}}
uses: marocchino/sticky-pull-request-comment@v2
with:
header: performance
GITHUB_TOKEN: ${{ secrets.gh_token }}
path: ${{ github.workspace }}/${{ env.UTILS_DIR }}/scripts/temp.md
- name: Get latest accuracy results
id: accuracy
if: ${{ github.event.action != 'closed' }}
run: |
cd ${{ env.TEST_RESULTS_PATH }}
ACCURACY=$(ls -dt accuracy* | head -1)
echo "last_test=$ACCURACY" >> $GITHUB_OUTPUT
- name: Create accuracy comment on PR
if: ${{ github.event_name != 'schedule' && github.event_name == 'pull_request' && github.event.action != 'closed'}}
uses: marocchino/sticky-pull-request-comment@v2
with:
header: accuracy
GITHUB_TOKEN: ${{ secrets.gh_token }}
path: ${{ env.TEST_RESULTS_PATH }}/${{ steps.accuracy.outputs.last_test }}/results.md
- name: Get latest report
id: last_report
if: ${{ github.event_name == 'schedule' }}
run: |
cd ${{ env.REPORTS_PATH }}
latest="$(readlink -f $(ls -tp | grep -v /$ | head -1))"
echo "latest=$latest" >> $GITHUB_OUTPUT
- name: Send mail
if: ${{ github.event_name == 'schedule' }}
uses: dawidd6/action-send-mail@v3
with:
server_address: smtp.gmail.com
server_port: 465
username: ${{ secrets.mail_user }}
password: ${{ secrets.mail_pass }}
subject: ${{ env.MAIL_SUBJECT }}
to: ${{ env.MAIL_TO }}
from: ${{ env.MAIL_FROM }}
secure: true
body: ${{ env.MAIL_BODY }}
cc: ${{ env.MAIL_CC }}
ignore_cert: true
attachments: ${{ steps.last_report.outputs.latest}}
priority: normal
- name : Checkout for backup
if: ${{ github.event_name == 'schedule' || ( github.event.action == 'closed' && github.event.pull_request.merged == true) }}
uses: actions/checkout@v3
with:
repository: ${{ inputs.performance_backup_repo }}
path: ${{ env.PERFORMANCE_DIR }}
token: ${{ secrets.gh_token }}
- name: Backup
if: ${{ github.event_name == 'schedule' || ( github.event.action == 'closed' && github.event.pull_request.merged == true) }}
run : |
cp -R $(ls -td -- ${{ env.TEST_RESULTS_PATH }}/perf*/ | head -n 1) $GITHUB_WORKSPACE/${{ env.PERFORMANCE_DIR }}/${{ inputs.organization }}
cd $GITHUB_WORKSPACE/${{ env.PERFORMANCE_DIR }}
git add .
git status
git config --local user.email github-actions@github.com
git config --local user.name github-actions
git commit -m "Backup latest perf results" -a
git push
- name: Clean merged PR data
if: ${{ github.event.action == 'closed' && github.event.pull_request.merged == true}}
run: >
docker run
-v $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts:/migraphx/sh:ro
-v ${{ env.MIGRAPHX_PATH }}:/data/migraphx/${{ inputs.organization }}
--workdir /migraphx/sh
rocm-migraphx:${{ inputs.rocm_release }} bash -c "./clean_after_pr.sh ${{ env.TEST_RESULTS_PATH }} ${{ env.PR_ID }} merged"
- name: Clean closed PR data
if: ${{ github.event.action == 'closed' && github.event.pull_request.merged == false}}
run: >
docker run
-v $GITHUB_WORKSPACE/${{ env.UTILS_DIR }}/scripts:/migraphx/sh:ro
-v ${{ env.MIGRAPHX_PATH }}:/data/migraphx/${{ inputs.organization }}
--workdir /migraphx/sh
rocm-migraphx:${{ inputs.rocm_release }} bash -c "./clean_after_pr.sh ${{ env.TEST_RESULTS_PATH }} ${{ env.PR_ID }} closed"