From c233c8373aa7dfabe75a68edd606b797786a2ced Mon Sep 17 00:00:00 2001 From: umarcor Date: Wed, 15 Dec 2021 05:39:18 +0100 Subject: [PATCH] ci: avoid installing the toolchain from scratch in each job * Use GitHub Packages * Use container in a step, instead of a container job F4PGA_INSTALL_DIR changed to an absolute location due to https://github.com/actions/runner/issues/1525 Signed-off-by: Unai Martinez-Corral --- ...e_job_matrices.py => generate_matrices.py} | 42 ++++++++++-- .github/workflows/Automerge.yml | 67 +++++++++++++++++-- .github/workflows/sphinx-tuttest.yml | 64 ++++++------------ docs/building-examples.rst | 2 +- docs/getting.rst | 2 +- 5 files changed, 122 insertions(+), 55 deletions(-) rename .github/scripts/{generate_job_matrices.py => generate_matrices.py} (69%) diff --git a/.github/scripts/generate_job_matrices.py b/.github/scripts/generate_matrices.py similarity index 69% rename from .github/scripts/generate_job_matrices.py rename to .github/scripts/generate_matrices.py index 18436d0..70f6d3b 100755 --- a/.github/scripts/generate_job_matrices.py +++ b/.github/scripts/generate_matrices.py @@ -19,6 +19,8 @@ from os import environ from sys import argv as sys_argv +registry = 'ghcr.io/chipsalliance/f4pga/dev/conda' + isFork = len(sys_argv)>1 and sys_argv[1] != 'chipsalliance/f4pga-examples' runs_on = ( @@ -86,21 +88,19 @@ def get_jobs( for osver in osvers: jobs.extend([{ - 'name': "Surelog" if usesSurelog else "Default", + 'name': ("Surelog" if usesSurelog else "Default") + f' | xc7 | {osver[0]}/{osver[1]} | {example}', 'runs-on': runs_on, 'fpga-fam': "xc7", - 'os': osver[0], - 'os-version': osver[1], + 'image': f'{registry}/{osver[0]}/{osver[1]}/xc7', 'example': example, 'surelog': "-parse -DSYNTHESIS" if usesSurelog else "" } for example in examples]) jobs.extend([{ - 'name': "Surelog" if usesSurelog else "Default", + 'name': ("Surelog" if usesSurelog else "Default") + f' | eos-s3 | {osver[0]}/{osver[1]} | counter', 'runs-on': runs_on, 'fpga-fam': "eos-s3", - 'os': osver[0], - 'os-version': osver[1], + 'image': f'{registry}/{osver[0]}/{osver[1]}/eos-s3', 'example': "counter", 'surelog': "-parse -DSYNTHESIS" if usesSurelog else "" } for osver in osvers]) @@ -111,3 +111,33 @@ for distribution in ['debian', 'ubuntu', 'fedora', 'centos']: jobs = get_jobs(distribution, False) + get_jobs(distribution, True) print(f'::set-output name={distribution}::{jobs!s}') print(f"{distribution}: {jobs!s}") + +utils = { + 'ubuntu': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG', + 'debian': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG', + 'centos': 'yum -y install wget patch', + 'fedora': 'dnf install -y wget patch' +} + +images = [ + { + 'registry': f'{registry}', + 'image': f'{osver[0]}/{osver[1]}/{fam}', + 'from': f'{osver[0]}:{osver[1]}', + 'utils': utils[osver[0]], + 'args': f'{fam} {osver[0]}' + } for osver in [ + ("debian", "buster"), + ("debian", "bullseye"), + ("debian", "sid"), + ("fedora", "35"), + ("fedora", "36"), + ("ubuntu", "xenial"), + ("ubuntu", "bionic"), + ("ubuntu", "focal"), + ("centos", "7") + ] for fam in ['xc7', 'eos-s3'] +] + +print(f'::set-output name=images::{images!s}') +print(f"images: {images!s}") diff --git a/.github/workflows/Automerge.yml b/.github/workflows/Automerge.yml index ad2e596..d8cb476 100644 --- a/.github/workflows/Automerge.yml +++ b/.github/workflows/Automerge.yml @@ -22,32 +22,89 @@ jobs: ubuntu: ${{ steps.generate.outputs.ubuntu }} fedora: ${{ steps.generate.outputs.fedora }} centos: ${{ steps.generate.outputs.centos }} + images: ${{ steps.generate.outputs.images }} steps: - uses: actions/checkout@v3 - id: generate - run: ./.github/scripts/generate_job_matrices.py '${{ github.repository }}' + run: ./.github/scripts/generate_matrices.py '${{ github.repository }}' + + + Packages: + needs: Matrices + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.Matrices.outputs.images) }} + name: 'Image: ${{ matrix.image }}' + env: + DOCKER_BUILDKIT: 1 + + steps: + + - name: Setup repository + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Build container image + run: | + docker build -t ${{ matrix.registry }}/${{ matrix.image }} . -f- <<'EOF' + FROM ${{ matrix.from }} + + ENV LANG en_US.UTF-8 + + # Install utils + RUN ${{ matrix.utils }} + + # Install tuttest + RUN wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest \ + && chmod a+rx /usr/bin/tuttest + + # Install F4PGA toolchain + RUN --mount=type=bind,target=/tmp/ctx cp -vr /tmp/ctx /tmp/repo \ + && cd /tmp/repo && bash /tmp/repo/.github/scripts/install-toolchain.sh ${{ matrix.args }} \ + && cd .. && rm -rf /tmp/repo + EOF + + - name: Push container image + if: ${{ github.event_name != 'pull_request' }} + uses: pyTooling/Actions/with-post-step@r0 + with: + main: | + echo '${{ github.token }}' | docker login ghcr.io -u GitHub-Actions --password-stdin + docker push ${{ matrix.registry }}/${{ matrix.image }} + post: docker logout ghcr.io Debian: - needs: Matrices + needs: + - Matrices + - Packages uses: ./.github/workflows/sphinx-tuttest.yml with: matrix: ${{ needs.Matrices.outputs.debian }} Ubuntu: - needs: Matrices + needs: + - Matrices + - Packages uses: ./.github/workflows/sphinx-tuttest.yml with: matrix: ${{ needs.Matrices.outputs.ubuntu }} Fedora: - needs: Matrices + needs: + - Matrices + - Packages uses: ./.github/workflows/sphinx-tuttest.yml with: matrix: ${{ needs.Matrices.outputs.fedora }} Centos: - needs: Matrices + needs: + - Matrices + - Packages uses: ./.github/workflows/sphinx-tuttest.yml with: matrix: ${{ needs.Matrices.outputs.centos }} diff --git a/.github/workflows/sphinx-tuttest.yml b/.github/workflows/sphinx-tuttest.yml index 3bd8064..abca622 100644 --- a/.github/workflows/sphinx-tuttest.yml +++ b/.github/workflows/sphinx-tuttest.yml @@ -22,16 +22,12 @@ jobs: matrix: include: ${{ fromJson(inputs.matrix) }} runs-on: ${{ matrix.runs-on }} - name: ${{ matrix.os-version }} | ${{ matrix.name }} | ${{ matrix.fpga-fam }} | ${{ matrix.example }} + name: ${{ matrix.name }} env: - LANG: "en_US.UTF-8" - DEBIAN_FRONTEND: "noninteractive" GHA_PREEMPTIBLE: "false" SURELOG_CMD: ${{ matrix.surelog }} - container: ${{matrix.os}}:${{matrix.os-version}} - steps: - name: Setup repository @@ -39,48 +35,32 @@ jobs: with: submodules: recursive - - name: Install utils - run: | - case ${{ matrix.os }} in - debian|ubuntu) apt -qqy update && apt -qqy install git wget locales && locale-gen $LANG ;; - centos) yum -y install git wget ;; - fedora) dnf install -y git wget ;; - esac - - - name: Install tuttest - run: | - wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest - chmod a+rx /usr/bin/tuttest + - name: Pull container image + run: docker pull ${{matrix.image}} - name: Patch package URLs if: inputs.latest == true run: | - case ${{ matrix.os }} in - debian|ubuntu) - apt update -qq - apt install -y patch - ;; - centos) - yum -y install patch - ;; - fedora) - dnf install -y patch - ;; - esac + sudo apt update -qq + sudo apt install -y patch patch -p1 -i .github/latest.patch - - name: Install F4PGA toolchain - run: bash .github/scripts/install-toolchain.sh ${{matrix.fpga-fam}} ${{matrix.os}} - - name: Build examples - run: bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}} + # Workaround for 'uses: docker://${{matrix.image}}' not being supported. + # See https://github.com/github/feedback/discussions/9049 + run: >- + docker run --rm + -v $(pwd):/wrk -w /wrk + -e DEBIAN_FRONTEND="noninteractive" + ${{matrix.image}} + bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}} - - uses: actions/upload-artifact@v3 - with: - name: f4pga-examples-bitstreams-${{ matrix.name }} - path: '**/*.bit' - - - uses: actions/upload-artifact@v3 - with: - name: f4pga-examples-plots-${{ matrix.name }} - path: '**/plot_*.svg' +# - uses: actions/upload-artifact@v3 +# with: +# name: f4pga-examples-bitstreams-${{ matrix.name }} +# path: '**/*.bit' +# +# - uses: actions/upload-artifact@v3 +# with: +# name: f4pga-examples-plots-${{ matrix.name }} +# path: '**/plot_*.svg' diff --git a/docs/building-examples.rst b/docs/building-examples.rst index c15734d..a8b66fb 100644 --- a/docs/building-examples.rst +++ b/docs/building-examples.rst @@ -9,7 +9,7 @@ set it to earlier, for example: .. code-block:: bash :name: export-install-dir - export F4PGA_INSTALL_DIR=~/opt/f4pga + export F4PGA_INSTALL_DIR=/opt/f4pga Select your FPGA family: diff --git a/docs/getting.rst b/docs/getting.rst index 4a4081f..9527381 100644 --- a/docs/getting.rst +++ b/docs/getting.rst @@ -84,7 +84,7 @@ and so you will need to add some ``sudo`` commands to the instructions below. .. code-block:: bash :name: conda-install-dir - export F4PGA_INSTALL_DIR=~/opt/f4pga + export F4PGA_INSTALL_DIR=/opt/f4pga Setup and download assets ~~~~~~~~~~~~~~~~~~~~~~~~~