From d70085d2a5685acbcc85afb521e185c8d5e6c185 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Tue, 23 Apr 2024 16:25:41 +0100 Subject: [PATCH] Magic Nix Cache --- .github/workflows/ci.yml | 246 +++++++++++++++++++++++++++++++++++---- 1 file changed, 225 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b4810e3..b37e63dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,28 +32,25 @@ jobs: - name: Sync repository uses: actions/checkout@v4 - - name: Retrieve persistent `/nix/store` - uses: actions/cache@v4 - with: - key: ${{ runner.os }}-nix-store }} - path: /nix/store + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@main - - name: Install Nix (with flakes and nix-command enabled) - uses: cachix/install-nix-action@v26 - with: - nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - experimental-features = nix-command flakes - extra-substituters = https://nix-community.cachix.org - extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= - extra-substituters = https://crane.cachix.org - extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= - extra-substituters = https://nix.computer.surgery/conduit - extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo= - extra-substituters = https://attic.kennel.juneis.dog/conduit - extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= - extra-substituters = https://attic.kennel.juneis.dog/conduwuit - extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= + - name: Configure Magic Nix Cache + uses: DeterminateSystems/magic-nix-cache-action@main + + - name: Apply custom Nix configuration + run: | + echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf + echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf + echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf + echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf + echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf + echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf + echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf + echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf + echo "extra-trusted-public-keys = conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=" >> /etc/nix/nix.conf + echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf + echo "extra-trusted-public-keys = conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=" >> /etc/nix/nix.conf - name: Add alternative Nix binary caches if specified if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} @@ -70,3 +67,210 @@ jobs: run: | nix develop --command true direnv allow + + - name: Run CI tests + run: | + direnv exec . engage + + - name: Pre-build x86_64 dependencies + run: > + bin/nix-build-and-cache + .#rocksdb-static-x86_64-unknown-linux-musl + .#conduit-deps-static-x86_64-unknown-linux-musl + + - name: Pre-build aarch64 dependencies + run: > + bin/nix-build-and-cache + .#rocksdb-static-aarch64-unknown-linux-musl + .#conduit-deps-static-aarch64-unknown-linux-musl + + - name: Build static targets + run: > + bin/nix-build-and-cache + .#static-x86_64-unknown-linux-musl + .#static-x86_64-unknown-linux-musl-jemalloc + .#static-aarch64-unknown-linux-musl + .#static-aarch64-unknown-linux-musl-jemalloc + + - name: Organise and prepare static outputs + run: | + mkdir -p target/debian + mv result/bin/conduit target/static-x86_64-unknown-linux-musl + direnv exec . cargo deb --no-build --no-strip --output target/debian/x86_64-unknown-linux-musl.deb + mv result-1/bin/conduit target/static-x86_64-unknown-linux-musl-jemalloc + direnv exec . cargo deb --no-build --no-strip --output target/debian/x86_64-unknown-linux-musl-jemalloc.deb + mv result-2/bin/conduit target/static-aarch64-unknown-linux-musl + direnv exec . cargo deb --no-build --no-strip --output target/debian/aarch64-unknown-linux-musl.deb + mv result-3/bin/conduit target/static-aarch64-unknown-linux-musl-jemalloc + direnv exec . cargo deb --no-build --no-strip --output target/debian/aarch64-unknown-linux-musl-jemalloc.deb + mv target/debian/*.deb target/ + rm -rf target/debian + + - name: Upload static-x86_64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: static-x86_64-unknown-linux-musl + path: target/static-x86_64-unknown-linux-musl + if-no-files-found: error + + - name: Upload static-x86_64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: static-x86_64-unknown-linux-musl-jemalloc + path: target/static-x86_64-unknown-linux-musl-jemalloc + if-no-files-found: error + + - name: Upload static-aarch64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: static-aarch64-unknown-linux-musl + path: target/static-aarch64-unknown-linux-musl + if-no-files-found: error + + - name: Upload static-aarch64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: static-aarch64-unknown-linux-musl-jemalloc + path: target/static-aarch64-unknown-linux-musl-jemalloc + if-no-files-found: error + + - name: Upload deb x86_64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: x86_64-unknown-linux-musl.deb + path: target/x86_64-unknown-linux-musl.deb + if-no-files-found: error + + - name: Upload deb x86_64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: x86_64-unknown-linux-musl-jemalloc.deb + path: target/x86_64-unknown-linux-musl-jemalloc.deb + if-no-files-found: error + + - name: Upload deb aarch64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: aarch64-unknown-linux-musl.deb + path: target/aarch64-unknown-linux-musl.deb + if-no-files-found: error + + - name: Upload deb aarch64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: aarch64-unknown-linux-musl-jemalloc.deb + path: target/aarch64-unknown-linux-musl-jemalloc.deb + if-no-files-found: error + + - name: Build OCI image targets + run: > + bin/nix-build-and-cache + .#oci-image-x86_64-unknown-linux-musl + .#oci-image-x86_64-unknown-linux-musl-jemalloc + .#oci-image-aarch64-unknown-linux-musl + .#oci-image-aarch64-unknown-linux-musl-jemalloc + + - name: Organise OCI image outputs + run: | + mv result oci-image-x86_64-unknown-linux-musl.tar.gz + mv result-1 oci-image-x86_64-unknown-linux-musl-jemalloc.tar.gz + mv result-2 oci-image-aarch64-unknown-linux-musl.tar.gz + mv result-3 oci-image-aarch64-unknown-linux-musl-jemalloc.tar.gz + + - name: Upload OCI image x86_64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-unknown-linux-musl.tar.gz + path: oci-image-x86_64-unknown-linux-musl.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image x86_64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-unknown-linux-musl-jemalloc.tar.gz + path: oci-image-x86_64-unknown-linux-musl-jemalloc.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image aarch64-unknown-linux-musl + uses: actions/upload-artifact@v4 + with: + name: oci-image-aarch64-unknown-linux-musl.tar.gz + path: oci-image-aarch64-unknown-linux-musl.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image aarch64-unknown-linux-musl-jemalloc + uses: actions/upload-artifact@v4 + with: + name: oci-image-aarch64-unknown-linux-musl-jemalloc.tar.gz + path: oci-image-aarch64-unknown-linux-musl-jemalloc.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Load OCI Images and tag + run: | + ID_AARCH64=$(docker load -i oci-image-aarch64-unknown-linux-musl-jemalloc.tar.gz | sed -n 's/Loaded image: \(.*\)/\1/p') + docker tag $ID_AARCH64 conduwuit:${{ github.sha }}-aarch64-jemalloc + ID_X86_64=$(docker load -i oci-image-x86_64-unknown-linux-musl-jemalloc.tar.gz | sed -n 's/Loaded image: \(.*\)/\1/p') + docker tag $ID_X86_64 conduwuit:${{ github.sha }}-x86_64-jemalloc + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push combined manifest to GitHub + run: | + REPO_GHCR="ghcr.io/${{ github.repository }}" + SHA_TAG="${{ github.ref_name }}-${{ github.sha }}" + BRANCH_TAG="${{ github.ref_name }}" + if [ "$BRANCH_TAG" == "main" ]; then + BRANCH_TAG="latest" + fi + + docker manifest create $REPO_GHCR:$SHA_TAG \ + --amend conduwuit:${{ github.sha }}-x86_64-jemalloc \ + --amend conduwuit:${{ github.sha }}-aarch64-jemalloc + docker manifest push $REPO_GHCR:$SHA_TAG + + docker manifest create $REPO_GHCR:$BRANCH_TAG \ + --amend conduwuit:${{ github.sha }}-x86_64-jemalloc \ + --amend conduwuit:${{ github.sha }}-aarch64-jemalloc + docker manifest push $REPO_GHCR:$BRANCH_TAG + + - name: Login to Docker Hub + uses: docker/login-action@v3 + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} + if: ${{ (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create and push combined manifest to Docker Hub + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }} + if: ${{ (env.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + run: | + REPO_DOCKER="docker.io/${{ github.repository }}" + SHA_TAG="${{ github.ref_name }}-${{ github.sha }}" + BRANCH_TAG="${{ github.ref_name }}" + if [ "$BRANCH_TAG" == "main" ]; then + BRANCH_TAG="latest" + fi + + docker manifest create $REPO_DOCKER:$SHA_TAG \ + --amend conduwuit:${{ github.sha }}-x86_64-jemalloc \ + --amend conduwuit:${{ github.sha }}-aarch64-jemalloc + docker manifest push $REPO_DOCKER:$SHA_TAG + + docker manifest create $REPO_DOCKER:$BRANCH_TAG \ + --amend conduwuit:${{ github.sha }}-x86_64-jemalloc \ + --amend conduwuit:${{ github.sha }}-aarch64-jemalloc + docker manifest push $REPO_DOCKER:$BRANCH_TAG