This repository has no description
👷 Setup video regression testing
author
Gwenn Le Bihan
date
(Oct 23, 2025, 1:25 PM +0200)
commit
925a6d6d
925a6d6d86fedcc37dfc7926b7dddf34f3cdf9ba
parent
1a694c08
1a694c08d23e49275bacfcda32cb91e8417c7c71
+165
-4
+146
-2
.github/workflows/check.yml
+146
-2
.github/workflows/check.yml
···
25
25
with:
26
26
file-url: "https://gwennlbh.github.io/shapemaker/timings.csv"
27
27
file-name: results-main.csv
28
-
# - uses: fontist/setup-fontist@v2
29
-
# - run: fontist install "Victor Mono"
30
28
- run: just timings results-main.csv
31
29
- id: video
32
30
uses: actions/upload-artifact@v4
···
86
84
- uses: Swatinem/rust-cache@v2
87
85
- uses: actions-rs/cargo@v1
88
86
with: { command: test, args: --workspace }
87
+
88
+
video-baseline-update:
89
+
if: github.event_name == 'push'
90
+
name: Update baseline test video
91
+
runs-on: ubuntu-latest
92
+
concurrency:
93
+
group: video-test-update-baseline-${{ github.ref_name }}
94
+
cancel-in-progress: true
95
+
permissions:
96
+
contents: write
97
+
defaults:
98
+
run:
99
+
working-directory: ./examples/schedule-hell
100
+
steps:
101
+
- uses: actions/checkout@v5
102
+
with:
103
+
ssh-key: ${{ secrets.AUTOCOMMIT_KEY }}
104
+
- name: Install FFMpeg
105
+
uses: FedericoCarboni/setup-ffmpeg@v3
106
+
- name: Install VMAF
107
+
uses: ethanjli/cached-download-action@v0.1.3
108
+
with:
109
+
url: https://github.com/Netflix/vmaf/releases/download/v3.0.0/vmaf
110
+
destination: /usr/local/bin/vmaf
111
+
- run: chmod +x /usr/local/bin/vmaf
112
+
- uses: actions-rs/toolchain@v1
113
+
with: { toolchain: stable }
114
+
- uses: Swatinem/rust-cache@v2
115
+
116
+
- run: mv baseline.mp4 old-baseline.mp4
117
+
- run: cargo run -- --duration 10 baseline.mp4
118
+
- name: Convert to yuv4mpeg
119
+
run: |
120
+
for f in $files; do
121
+
ffmpeg -i $f -f yuv4mpegpipe -y $(basename $f .mp4).y4m
122
+
done
123
+
env:
124
+
files: old-baseline.mp4 baseline.mp4
125
+
- name: Run VMAF
126
+
run: vmaf --json --output comparison.json
127
+
--reference old-baseline.y4m
128
+
--distorted baseline.y4m
129
+
- name: Judge score
130
+
id: results
131
+
run: |
132
+
score=$(jq .pooled_metrics.vmaf.mean comparison.json)
133
+
134
+
if (( $(echo "$score < 95" | bc -l) )); then
135
+
conclusion=changed
136
+
else
137
+
conclusion=unchanged
138
+
fi
139
+
140
+
echo "score=$score" >> $GITHUB_OUTPUT
141
+
echo "conclusion=$conclusion" >> $GITHUB_OUTPUT
142
+
143
+
- if: steps.results.outputs.conclusion == 'changed'
144
+
name: Commit changes
145
+
uses: stefanzweifel/git-auto-commit-action@v7
146
+
with:
147
+
commit_message: ✅ Update video test baseline
148
+
file_pattern: examples/schedule-hell/baseline.mp4
149
+
video:
150
+
if: github.event_name == 'pull_request'
151
+
name: Video
152
+
runs-on: ubuntu-latest
153
+
permissions:
154
+
checks: write
155
+
contents: read
156
+
concurrency:
157
+
group: video-test-${{ github.ref_name }}
158
+
cancel-in-progress: true
159
+
defaults:
160
+
run:
161
+
working-directory: ./examples/schedule-hell
162
+
steps:
163
+
- uses: actions/checkout@v5
164
+
- name: Install FFMpeg
165
+
uses: FedericoCarboni/setup-ffmpeg@v3
166
+
- name: Install VMAF
167
+
uses: ethanjli/cached-download-action@v0.1.3
168
+
with:
169
+
url: https://github.com/Netflix/vmaf/releases/download/v3.0.0/vmaf
170
+
destination: /usr/local/bin/vmaf
171
+
- run: chmod +x /usr/local/bin/vmaf
172
+
- uses: actions-rs/toolchain@v1
173
+
with: { toolchain: stable }
174
+
- uses: Swatinem/rust-cache@v2
175
+
- uses: extractions/setup-just@v3
176
+
- run: cargo run -- --duration 10 result.mp4
177
+
- name: Convert to yuv4mpeg
178
+
run: |
179
+
for f in $files; do
180
+
ffmpeg -i $f -f yuv4mpegpipe -y $(basename $f .mp4).y4m
181
+
done
182
+
env:
183
+
files: baseline.mp4 result.mp4
184
+
185
+
- name: Compare with VMAF
186
+
run: vmaf --json
187
+
--output comparison.json
188
+
--reference baseline.y4m
189
+
--distorted result.y4m
190
+
191
+
- name: Judge score
192
+
id: results
193
+
run: |
194
+
score=$(jq .pooled_metrics.vmaf.mean comparison.json)
195
+
196
+
if (( $(echo "$score < 95" | bc -l) )); then
197
+
conclusion=failure
198
+
comparison_display="$score < 95"
199
+
else
200
+
conclusion=success
201
+
comparison_display="$score ≥ 95"
202
+
fi
203
+
204
+
echo "score=$score" >> $GITHUB_OUTPUT
205
+
echo "conclusion=$conclusion" >> $GITHUB_OUTPUT
206
+
echo "comparison_display=$comparison_display" >> $GITHUB_OUTPUT
207
+
208
+
- name: Upload comparison artifacts
209
+
uses: actions/upload-artifact@v4
210
+
id: artifacts
211
+
with:
212
+
path: |
213
+
examples/schedule-hell/comparison.json
214
+
examples/schedule-hell/baseline.mp4
215
+
examples/schedule-hell/result.mp4
216
+
217
+
- uses: LouisBrunner/checks-action@v2
218
+
with:
219
+
name: Compare to baseline test video
220
+
token: ${{ secrets.GITHUB_TOKEN }}
221
+
conclusion: ${{ steps.results.outputs.conclusion }}
222
+
output: |
223
+
{
224
+
"title": "VMAF Score is ${{ steps.results.outputs.comparison_display }}",
225
+
"summary": "Artifacts available at ${{ steps.artifacts.outputs.artifact-url }}"
226
+
}
227
+
228
+
- name: Fail if score too low
229
+
if: steps.results.outputs.conclusion == 'failure'
230
+
run: exit 1
231
+
232
+
+5
examples/schedule-hell/.gitignore
+5
examples/schedule-hell/.gitignore
examples/schedule-hell/baseline.mp4
examples/schedule-hell/baseline.mp4
This is a binary file and will not be displayed.
examples/schedule-hell/schedule-hell-exerpt.mp4
examples/schedule-hell/schedule-hell-exerpt.mp4
This is a binary file and will not be displayed.
+4
-1
examples/schedule-hell/src/main.rs
+4
-1
examples/schedule-hell/src/main.rs
+10
-1
renovate.json
+10
-1
renovate.json
···
1
1
{
2
2
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
3
-
"extends": ["github>gwennlbh/renovate"]
3
+
"extends": ["github>gwennlbh/renovate"],
4
+
"customManagers": [
5
+
{
6
+
"customType": "regex",
7
+
"managerFilePatterns": [".github/workflows/*.yml"],
8
+
"matchStrings": ["https://github.com/Netflix/vmaf/releases/download/(?<currentValue>\\w+)/vmaf"],
9
+
"depNameTemplate": "Netflix/vmaf",
10
+
"datasourceTemplate": "github-releases"
11
+
}
12
+
]
4
13
}