ci: avoid installing the toolchain from scratch in each job

* Use GitHub Packages * Use container in a step, instead of a container job F4PGA_INSTALL_DIR changed to an absolute location due to https://github.com/actions/runner/issues/1525 Signed-off-by: Unai Martinez-Corral <umartinezcorral@antmicro.com>
2021-12-15 05:39:18 +01:00 · 2021-12-15 05:39:18 +01:00 · c233c8373a
parent 524dfb6e74
commit c233c8373a
5 changed files with 122 additions and 55 deletions
--- a/.github/scripts/generate_job_matrices.py
+++ b/.github/scripts/generate_job_matrices.py
@ -19,6 +19,8 @@
 from os import environ
 from sys import argv as sys_argv

+registry = 'ghcr.io/chipsalliance/f4pga/dev/conda'
+
 isFork = len(sys_argv)>1 and sys_argv[1] != 'chipsalliance/f4pga-examples'

 runs_on = (
@ -86,21 +88,19 @@ def get_jobs(

    for osver in osvers:
        jobs.extend([{
-            'name': "Surelog" if usesSurelog else "Default",
+            'name': ("Surelog" if usesSurelog else "Default") + f' | xc7 | {osver[0]}/{osver[1]} | {example}',
            'runs-on': runs_on,
            'fpga-fam': "xc7",
-            'os': osver[0],
-            'os-version': osver[1],
+            'image': f'{registry}/{osver[0]}/{osver[1]}/xc7',
            'example': example,
            'surelog': "-parse -DSYNTHESIS" if usesSurelog else ""
        } for example in examples])

    jobs.extend([{
-        'name': "Surelog" if usesSurelog else "Default",
+        'name': ("Surelog" if usesSurelog else "Default") + f' | eos-s3 | {osver[0]}/{osver[1]} | counter',
        'runs-on': runs_on,
        'fpga-fam': "eos-s3",
-        'os': osver[0],
-        'os-version': osver[1],
+        'image': f'{registry}/{osver[0]}/{osver[1]}/eos-s3',
        'example': "counter",
        'surelog': "-parse -DSYNTHESIS" if usesSurelog else ""
    } for osver in osvers])
@ -111,3 +111,33 @@ for distribution in ['debian', 'ubuntu', 'fedora', 'centos']:
    jobs = get_jobs(distribution, False) + get_jobs(distribution, True)
    print(f'::set-output name={distribution}::{jobs!s}')
    print(f"{distribution}: {jobs!s}")
+
+utils = {
+    'ubuntu': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG',
+    'debian': 'apt -qqy update && apt -qqy install wget locales patch && locale-gen $LANG',
+    'centos': 'yum -y install wget patch',
+    'fedora': 'dnf install -y wget patch'
+}
+
+images = [
+    {
+        'registry': f'{registry}',
+        'image': f'{osver[0]}/{osver[1]}/{fam}',
+        'from': f'{osver[0]}:{osver[1]}',
+        'utils': utils[osver[0]],
+        'args': f'{fam} {osver[0]}'
+    } for osver in [
+        ("debian", "buster"),
+        ("debian", "bullseye"),
+        ("debian", "sid"),
+        ("fedora", "35"),
+        ("fedora", "36"),
+        ("ubuntu", "xenial"),
+        ("ubuntu", "bionic"),
+        ("ubuntu", "focal"),
+        ("centos", "7")
+    ] for fam in ['xc7', 'eos-s3']
+]
+
+print(f'::set-output name=images::{images!s}')
+print(f"images: {images!s}")
--- a/.github/workflows/Automerge.yml
+++ b/.github/workflows/Automerge.yml
@ -22,32 +22,89 @@ jobs:
      ubuntu: ${{ steps.generate.outputs.ubuntu }}
      fedora: ${{ steps.generate.outputs.fedora }}
      centos: ${{ steps.generate.outputs.centos }}
+      images: ${{ steps.generate.outputs.images }}
    steps:
      - uses: actions/checkout@v3
      - id: generate
-        run: ./.github/scripts/generate_job_matrices.py '${{ github.repository }}'
+        run: ./.github/scripts/generate_matrices.py '${{ github.repository }}'
+
+
+  Packages:
+    needs: Matrices
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.Matrices.outputs.images) }}
+    name: 'Image: ${{ matrix.image }}'
+    env:
+      DOCKER_BUILDKIT: 1
+
+    steps:
+
+      - name: Setup repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Build container image
+        run: |
+          docker build -t ${{ matrix.registry }}/${{ matrix.image }} . -f- <<'EOF'
+          FROM ${{ matrix.from }}
+
+          ENV LANG en_US.UTF-8
+
+          # Install utils
+          RUN ${{ matrix.utils }}
+
+          # Install tuttest
+          RUN wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest \
+           && chmod a+rx /usr/bin/tuttest
+
+          # Install F4PGA toolchain
+          RUN --mount=type=bind,target=/tmp/ctx cp -vr /tmp/ctx /tmp/repo \
+           && cd /tmp/repo && bash /tmp/repo/.github/scripts/install-toolchain.sh ${{ matrix.args }} \
+           && cd .. && rm -rf /tmp/repo
+          EOF
+
+      - name: Push container image
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: pyTooling/Actions/with-post-step@r0
+        with:
+          main: |
+            echo '${{ github.token }}' | docker login ghcr.io -u GitHub-Actions --password-stdin
+            docker push ${{ matrix.registry }}/${{ matrix.image }}
+          post: docker logout ghcr.io


  Debian:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
    uses: ./.github/workflows/sphinx-tuttest.yml
    with:
      matrix: ${{ needs.Matrices.outputs.debian }}

  Ubuntu:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
    uses: ./.github/workflows/sphinx-tuttest.yml
    with:
      matrix: ${{ needs.Matrices.outputs.ubuntu }}

  Fedora:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
    uses: ./.github/workflows/sphinx-tuttest.yml
    with:
      matrix: ${{ needs.Matrices.outputs.fedora }}

  Centos:
-    needs: Matrices
+    needs:
+      - Matrices
+      - Packages
    uses: ./.github/workflows/sphinx-tuttest.yml
    with:
      matrix: ${{ needs.Matrices.outputs.centos }}
--- a/.github/workflows/sphinx-tuttest.yml
+++ b/.github/workflows/sphinx-tuttest.yml
@ -22,16 +22,12 @@ jobs:
      matrix:
        include: ${{ fromJson(inputs.matrix) }}
    runs-on: ${{ matrix.runs-on }}
-    name: ${{ matrix.os-version }} | ${{ matrix.name }} | ${{ matrix.fpga-fam }} | ${{ matrix.example }}
+    name: ${{ matrix.name }}

    env:
-      LANG: "en_US.UTF-8"
-      DEBIAN_FRONTEND: "noninteractive"
      GHA_PREEMPTIBLE: "false"
      SURELOG_CMD: ${{ matrix.surelog }}

-    container: ${{matrix.os}}:${{matrix.os-version}}
-
    steps:

      - name: Setup repository
@ -39,48 +35,32 @@ jobs:
        with:
          submodules: recursive

-      - name: Install utils
-        run: |
-          case ${{ matrix.os }} in
-            debian|ubuntu) apt -qqy update && apt -qqy install git wget locales && locale-gen $LANG ;;
-            centos) yum -y install git wget ;;
-            fedora) dnf install -y git wget ;;
-          esac
-
-      - name: Install tuttest
-        run: |
-          wget https://github.com/antmicro/tuttest/releases/download/v0.2-beta/tuttest -O /usr/bin/tuttest
-          chmod a+rx /usr/bin/tuttest
+      - name: Pull container image
+        run: docker pull ${{matrix.image}}

      - name: Patch package URLs
        if: inputs.latest == true
        run: |
-          case ${{ matrix.os }} in
-            debian|ubuntu)
-              apt update -qq
-              apt install -y patch
-            ;;
-            centos)
-              yum -y install patch
-            ;;
-            fedora)
-              dnf install -y patch
-            ;;
-          esac
+          sudo apt update -qq
+          sudo apt install -y patch
          patch -p1 -i .github/latest.patch

-      - name: Install F4PGA toolchain
-        run: bash .github/scripts/install-toolchain.sh ${{matrix.fpga-fam}} ${{matrix.os}}
-
      - name: Build examples
-        run: bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}}
+        # Workaround for 'uses: docker://${{matrix.image}}' not being supported.
+        # See https://github.com/github/feedback/discussions/9049
+        run: >-
+          docker run --rm
+          -v $(pwd):/wrk -w /wrk
+          -e DEBIAN_FRONTEND="noninteractive"
+          ${{matrix.image}}
+          bash .github/scripts/build-examples.sh ${{matrix.fpga-fam}} ${{matrix.example}}

-      - uses: actions/upload-artifact@v3
-        with:
-          name: f4pga-examples-bitstreams-${{ matrix.name }}
-          path: '**/*.bit'
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: f4pga-examples-plots-${{ matrix.name }}
-          path: '**/plot_*.svg'
+#      - uses: actions/upload-artifact@v3
+#        with:
+#          name: f4pga-examples-bitstreams-${{ matrix.name }}
+#          path: '**/*.bit'
+#
+#      - uses: actions/upload-artifact@v3
+#        with:
+#          name: f4pga-examples-plots-${{ matrix.name }}
+#          path: '**/plot_*.svg'
--- a/docs/building-examples.rst
+++ b/docs/building-examples.rst
@ -9,7 +9,7 @@ set it to earlier, for example:
 .. code-block:: bash
   :name: export-install-dir

-   export F4PGA_INSTALL_DIR=~/opt/f4pga
+   export F4PGA_INSTALL_DIR=/opt/f4pga

 Select your FPGA family:

--- a/docs/getting.rst
+++ b/docs/getting.rst
@ -84,7 +84,7 @@ and so you will need to add some ``sudo`` commands to the instructions below.
 .. code-block:: bash
   :name: conda-install-dir

-   export F4PGA_INSTALL_DIR=~/opt/f4pga
+   export F4PGA_INSTALL_DIR=/opt/f4pga

 Setup and download assets
 ~~~~~~~~~~~~~~~~~~~~~~~~~