.github/workflows/e2e_lighthouse_performance_acceptance_tests.yml
name: End-to-End, Lighthouse CI performance and Acceptance tests
on:
merge_group:
types: [checks_requested]
push:
branches:
- develop
- release-*
pull_request:
branches:
- develop
- release-*
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.8.15'
architecture: 'x64'
cache: 'pip'
cache-dependency-path: |
requirements.in
requirements.txt
requirements_dev.in
requirements_dev.txt
- uses: ./.github/actions/merge
- name: Cache node modules and third_party/static
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: |
/home/runner/work/oppia/yarn_cache
/home/runner/work/oppia/oppia/third_party/static
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock', 'dependencies.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- uses: ./.github/actions/install-oppia-dependencies
- name: Install chrome
run: python -m scripts.install_chrome_for_ci
- name: Install ffmpeg
run: sudo apt install ffmpeg
- name: Build Webpack
run: python -m scripts.build --prod_env
- name: Zip build files
# We avoid using ../ or absolute paths because unzip treats these as
# security issues and will refuse to follow them.
run: |
zip -rqy build_files.zip oppia/third_party oppia_tools oppia/build oppia/webpack_bundles oppia/proto_files oppia/app.yaml oppia/assets/hashes.json oppia/proto_files oppia/extensions/classifiers/proto/* oppia/backend_prod_files oppia/dist
working-directory: /home/runner/work/oppia
- name: Upload build files artifact
uses: actions/upload-artifact@v3
with:
name: build_files
path: /home/runner/work/oppia/build_files.zip
retention-days: 1
e2e_test:
needs: build
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
suite:
- name: accessibility
- name: additionalEditorFeatures
- name: additionalEditorFeaturesModals
- name: additionalPlayerFeatures
- name: adminPage
- name: blogDashboard
- name: blog
- name: checkpointFeatures
- name: classroomPage
- name: classroomPageFileUploadFeatures
- name: collections
- name: contributorDashboard
- name: contributorAdminDashboard
- name: coreEditorAndPlayerFeatures
- name: creatorDashboard
- name: embedding
- name: explorationFeedbackTab
- name: explorationImprovementsTab
- name: explorationHistoryTab
- name: explorationStatisticsTab
- name: explorationTranslationTab
- name: extensions
- name: featureGating
- name: feedbackUpdates
- name: fileUploadExtensions
- name: fileUploadFeatures
flaky_indicators: |
Play record button is not clickable
Translation tab button is not clickable
- name: learner
- name: learnerDashboard
- name: library
- name: navigation
- name: playVoiceovers
- name: preferences
- name: profileFeatures
- name: profileMenu
- name: publication
- name: skillEditor
- name: subscriptions
- name: topicsAndSkillsDashboard
- name: topicAndStoryEditor
- name: topicAndStoryEditorFileUploadFeatures
- name: topicAndStoryViewer
- name: users
- name: wipeout
name: e2e_test (${{ matrix.os }}, ${{ matrix.suite.name }})
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.8.15'
architecture: 'x64'
cache: 'pip'
cache-dependency-path: |
requirements.in
requirements.txt
requirements_dev.in
requirements_dev.txt
- uses: ./.github/actions/merge
- name: Cache node modules
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/yarn_cache
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Attempt to download build files artifact
id: download_artifact_1
uses: actions/download-artifact@v3
continue-on-error: true
with:
name: build_files
path: /home/runner/work/oppia/
- name: Unzip build files
if: steps.download_artifact_1.outcome != 'failure'
run: |
ls -la
unzip build_files.zip
rm build_files.zip
echo "Files in ./:"
ls -la .
echo "Files in oppia_tools:"
ls -la oppia_tools
echo "Files in oppia:"
ls -la oppia
echo "Files in build:"
ls -la oppia/build
echo "Files in third_party:"
ls -la oppia/third_party
working-directory: /home/runner/work/oppia
shell: bash
- uses: ./.github/actions/install-oppia-dependencies
- name: Install chrome
run: python -m scripts.install_chrome_for_ci
- name: Install ffmpeg
run: sudo apt install ffmpeg
- name: (Artifact re-build) Cache third_party/static
if: steps.download_artifact_1.outcome == 'failure'
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/oppia/third_party/static
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: (Artifact re-build) Build Webpack
if: steps.download_artifact_1.outcome == 'failure'
run: python -m scripts.build --prod_env
- name: Run E2E Test ${{ matrix.suite.name }}
uses: oppia/retry@a9fb265410e634c84ee90f3e87f323fde0541037
with:
max_attempts: 2
substrings_indicating_flaky_execution: ${{ matrix.suite.flaky_indicators || '' }}
command: >
VIDEO_RECORDING_IS_ENABLED=0;
xvfb-run -a --server-args="-screen 0, 1285x1000x24"
python -m scripts.run_e2e_tests --skip-install
--skip-build --suite=${{ matrix.suite.name }} --prod_env
- name: Uploading webdriverio-video as Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: webdriverio-video
path: /home/runner/work/oppia/webdriverio-video
- name: Uploading webdriverio screenshots as Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: webdriverio-screenshots
path: /home/runner/work/oppia/webdriverio-screenshots
- name: Uploading webpack bundles as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: webpack-bundles
path: /home/runner/work/oppia/oppia/build
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "An E2E test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
lighthouse:
needs: build
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
shard: [1, 2]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.8.15'
architecture: 'x64'
- uses: ./.github/actions/merge
- name: Cache node modules
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/yarn_cache
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Attempt to download build files artifact
id: download_artifact_1
uses: actions/download-artifact@v3
continue-on-error: true
with:
name: build_files
path: /home/runner/work/oppia/
- name: Unzip build files
if: steps.download_artifact_1.outcome != 'failure'
run: |
ls -la
unzip build_files.zip
rm build_files.zip
echo "Files in ./:"
ls -la .
echo "Files in oppia_tools:"
ls -la oppia_tools
echo "Files in oppia:"
ls -la oppia
echo "Files in build:"
ls -la oppia/build
echo "Files in third_party:"
ls -la oppia/third_party
working-directory: /home/runner/work/oppia
shell: bash
- name: (Artifact re-build) Cache third_party/static
if: steps.download_artifact_1.outcome == 'failure'
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/oppia/third_party/static
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: (Artifact re-build) Build Webpack
if: steps.download_artifact_1.outcome == 'failure'
run: python -m scripts.build --prod_env
- uses: ./.github/actions/install-oppia-dependencies
- name: Install chrome
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
run: python -m scripts.install_chrome_for_ci
- name: Run Lighthouse performance checks shard
if: startsWith(github.head_ref, 'update-changelog-for-release') == false
run: python -m scripts.run_lighthouse_tests --mode performance --shard ${{ matrix.shard }} --skip_build --record_screen
- name: Uploading puppeteer video as Artifacts
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: lhci-puppeteer-video
path: /home/runner/work/oppia/lhci-puppeteer-video/video.mp4
- name: Report failure if failed on oppia/oppia develop branch
if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
uses: ./.github/actions/send-webhook-notification
with:
message: "A Lighthouse test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
acceptance_test:
needs: build
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
suite:
- blog-admin/assign-roles-to-users-and-change-tag-properties
- blog-editor/try-to-publish-a-duplicate-blog-post-and-get-blocked
- curriculum-admin/create-and-publish-topics-and-stories
- exploration-editor/load-complete-and-restart-exploration-preview
- exploration-editor/create-exploration-and-change-basic-settings
- logged-out-user/click-all-buttons-on-about-page
- logged-out-user/click-all-buttons-on-about-foundation-page
- logged-out-user/click-all-buttons-on-donation-thanks-page
- logged-out-user/click-all-buttons-on-navbar
- logged-out-user/click-all-links-in-about-oppia-footer
- logged-out-user/click-all-links-on-get-started-page
- logged-in-user/subscribe-to-creator-and-view-all-explorations-by-that-creator
- practice-question-admin/add-and-remove-contribution-rights
- translation-admin/add-and-remove-translation-rights
- voiceover-admin/add-voiceover-artist-to-an-exploration
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
with:
python-version: '3.8.15'
architecture: 'x64'
cache: 'pip'
cache-dependency-path: |
requirements.in
requirements.txt
requirements_dev.in
requirements_dev.txt
- uses: ./.github/actions/merge
- name: Cache node modules
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/yarn_cache
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Attempt to download build files artifact
id: download_artifact_1
uses: actions/download-artifact@v3
continue-on-error: true
with:
name: build_files
path: /home/runner/work/oppia/
- name: Unzip build files
if: steps.download_artifact_1.outcome != 'failure'
run: |
ls -la
unzip build_files.zip
rm build_files.zip
echo "Files in ./:"
ls -la .
echo "Files in oppia_tools:"
ls -la oppia_tools
echo "Files in oppia:"
ls -la oppia
echo "Files in build:"
ls -la oppia/build
echo "Files in third_party:"
ls -la oppia/third_party
working-directory: /home/runner/work/oppia
shell: bash
- uses: ./.github/actions/install-oppia-dependencies
- name: (Artifact re-build) Cache third_party/static
if: steps.download_artifact_1.outcome == 'failure'
uses: actions/cache@v3
env:
cache-name: cache-node-modules
with:
path: /home/runner/work/oppia/oppia/third_party/static
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: (Artifact re-build) Build Webpack
if: steps.download_artifact_1.outcome == 'failure'
run: python -m scripts.build --prod_env
- name: Run Desktop Acceptance Test ${{ matrix.suite }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite }} --prod_env
- name: Run Mobile Acceptance Test ${{ matrix.suite }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite }} --prod_env --mobile
- name: Uploading webpack bundles as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: webpack-bundles
path: /home/runner/work/oppia/oppia/build
- name: Report failure if failed on oppia/oppia develop branch
if: ${{
failure() &&
github.event_name == 'push' &&
github.repository == 'oppia/oppia' &&
github.ref == 'refs/heads/develop'
}}
uses: ./.github/actions/send-webhook-notification
with:
message: "An acceptance test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}