Build, Sign & Notarise macOS builds (#486)

.github/workflows/build_and_deploy.yaml

@@ -0,0 +1,77 @@
+name: Build and Deploy
+on:
+  # Nightly build
+  schedule:
+    - cron: '0 9 * * *'
+  # Manual nightly & release
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: What type of build to trigger. Release builds should be ran from the `master` branch.
+        required: true
+        default: nightly
+        type: choice
+        options:
+          - nightly
+          - release
+      macos:
+        description: Whether to build macOS
+        required: true
+        type: boolean
+        default: true
+      deploy:
+        description: Whether to deploy artifacts
+        required: true
+        type: boolean
+        default: true
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+env:
+  # XXX: UPDATE THIS BEFORE WHEN GOING LIVE
+  R2_BUCKET: 'packages-element-io-test'
+jobs:
+  prepare:
+    uses: ./.github/workflows/build_prepare.yaml
+    with:
+      config: element.io/${{ inputs.mode || 'nightly' }}
+      version: ${{ inputs.mode == 'release' && '' || 'develop' }}
+      calculate-nightly-versions: ${{ inputs.mode != 'release' }}
+    secrets:
+      CF_R2_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+      CF_R2_TOKEN: ${{ secrets.CF_R2_TOKEN }}
+      CF_R2_S3_API: ${{ secrets.CF_R2_S3_API }}
+
+  macos:
+    if: github.event_name != 'workflow_dispatch' || inputs.macos
+    needs: prepare
+    name: macOS
+    uses: ./.github/workflows/build_macos.yaml
+    secrets: inherit
+    with:
+      sign: true
+      deploy-mode: true
+      base-url: https://packages.element.io/${{ inputs.mode == 'release' && 'desktop' || 'nightly' }}
+      version: ${{ needs.prepare.outputs.macos-version }}
+
+  deploy:
+    needs:
+      - macos
+    runs-on: ubuntu-latest
+    name: Deploy
+    if: always() && (github.event != 'workflow_dispatch' || inputs.deploy)
+    environment: packages.element.io
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: packages.element.io
+          path: packages.element.io
+
+      - name: Deploy artifacts
+        run: aws s3 cp --recursive packages.element.io/ s3://$R2_BUCKET/$DEPLOYMENT_DIR --endpoint-url $R2_URL --region auto
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+          R2_URL: ${{ secrets.CF_R2_S3_API }}
+          DEPLOYMENT_DIR: ${{ inputs.mode == 'release' && 'desktop' || 'nightly' }}

.github/workflows/build_and_test.yaml

@@ -2,7 +2,7 @@ name: Build and Test
 on:
   pull_request: {}
   push:
-    branches: [develop, master]
+    branches: [develop, staging, master]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true

.github/workflows/build_macos.yaml

@@ -3,9 +3,38 @@
 # the correct cache scoping, and additional care must be taken to not run untrusted actions on the develop branch.
 on:
     workflow_call:
+        secrets:
+            APPLE_ID:
+                required: false
+            APPLE_ID_PASSWORD:
+                required: false
+            APPLE_TEAM_ID:
+                required: false
+            APPLE_CSC_KEY_PASSWORD:
+                required: false
+            APPLE_CSC_LINK:
+                required: false
+        inputs:
+            version:
+                type: string
+                required: false
+                description: "Version string to override the one in package.json, used for non-release builds"
+            sign:
+                type: string
+                required: false
+                description: "Whether to sign & notarise the build, requires 'packages.element.io' environment"
+            deploy-mode:
+                type: string
+                required: false
+                description: "Whether to arrange artifacts in the arrangement needed for deployment, skipping unrelated ones"
+            base-url:
+                type: string
+                required: false
+                description: "The URL to which the output will be deployed, required if deploy-mode is enabled."
 jobs:
     build:
         runs-on: macos-latest
+        environment: ${{ inputs.sign && 'packages.element.io' || '' }}
         steps:
             - uses: actions/checkout@v3
 
@@ -40,12 +69,68 @@ jobs:
               if: steps.cache.outputs.cache-hit != 'true'
               run: "yarn build:native:universal"
 
+            - name: '[Nightly] Resolve version'
+              id: nightly
+              if: inputs.version != ''
+              run: |
+                  echo "config-args=--nightly '${{ inputs.version }}'" >> $GITHUB_OUTPUT
+
             - name: Build App
-              run: "yarn build:universal --publish never"
+              run: |
+                  scripts/generate-builder-config.ts ${{ steps.nightly.outputs.config-args }}
+                  yarn build:universal --publish never --config electron-builder.json
+              env:
+                  NOTARIZE_APPLE_ID: ${{ secrets.APPLE_ID }}
+                  NOTARIZE_APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+                  NOTARIZE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+                  CSC_KEY_PASSWORD: ${{ secrets.APPLE_CSC_KEY_PASSWORD }}
+                  CSC_LINK: ${{ secrets.APPLE_CSC_LINK }}
+
+            - name: Prepare artifacts for deployment
+              if: inputs.deploy-mode
+              run: |
+                  mv dist _dist
+                  mkdir -p dist/install/macos dist/update/macos
+                  mv _dist/*-mac.zip dist/update/macos/
+                  mv _dist/*.dmg dist/install/macos/
+        
+                  PKG_JSON_VERSION=$(cat package.json | jq -r .version)
+                  LATEST=$(find dist -type f -iname "*-mac.zip" | xargs -0 -n1 -- basename)
+                  URL="${{ inputs.base-url }}/update/macos/$LATEST"
+        
+                  jq -n --arg version "${VERSION:-$PKG_JSON_VERSION}" --arg url "$URL" '
+                    {
+                      currentRelease: $version,
+                      releases: [{
+                        version: $version,
+                        updateTo: {
+                          version: $version,
+                          url: $url,
+                        },
+                      }],
+                    }
+                  ' > dist/update/macos/releases.json
+                  jq -n --arg url "$URL" '
+                    { url: $url }
+                  ' > dist/update/macos/releases-legacy.json
+              env:
+                  VERSION: ${{ inputs.version }}
+
+            # We don't wish to store the installer for every nightly ever, so we only keep the latest
+            - name: '[Nightly] Strip version from installer file'
+              if: inputs.deploy-mode && inputs.version != ''
+              run: |
+                  mv dist/install/macos/*.dmg "dist/install/macos/Element Nightly.dmg"
+
+            - name: '[Release] Prepare release latest symlink'
+              if: inputs.deploy-mode && inputs.version == ''
+              run: |
+                  LATEST=$(find dist -type f -iname "*.dmg" | xargs -0 -n1 -- basename)
+                  ln -s "dist/install/macos/$LATEST" dist/install/macos/Element.dmg
 
             - name: Upload Artifacts
               uses: actions/upload-artifact@v3
               with:
-                  name: macos
+                  name: ${{ inputs.deploy-mode && 'packages.element.io' || 'macos' }}
                   path: dist
                   retention-days: 1

.github/workflows/build_prepare.yaml

@@ -1,3 +1,4 @@
+# This action helps perform common actions before the build_* actions are started in parallel.
 on:
     workflow_call:
         inputs:
@@ -9,10 +10,31 @@ on:
                 type: string
                 required: false
                 description: "The version tag to fetch, or 'develop', will pick automatically if not passed"
+            calculate-nightly-versions:
+                type: string
+                required: false
+                description: "Whether to calculate the version strings new Nightly builds should use"
+        secrets:
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_ACCESS_KEY_ID:
+                required: false
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_TOKEN:
+                required: false
+            # Required if `calculate-nightly-versions` is set
+            CF_R2_S3_API:
+                required: false
+        outputs:
+            macos-version:
+                description: "The version string the next macOS Nightly should use, only output for calculate-nightly-versions"
+                value: ${{ jobs.prepare.outputs.macos-version }}
 jobs:
     prepare:
         name: Prepare
+        environment: ${{ inputs.calculate-nightly-versions && 'packages.element.io' || '' }}
         runs-on: ubuntu-latest
+        outputs:
+            macos-version: ${{ steps.macos.outputs.version }}
         steps:
             - uses: actions/checkout@v3
 
@@ -41,3 +63,14 @@ jobs:
                       package.json
                       electronVersion
                       hakDependencies.json
+
+            - name: Calculate macOS Nightly version
+              id: macos
+              if: inputs.calculate-nightly-versions
+              run: |
+                  LATEST=$(aws s3 cp s3://$R2_BUCKET/nightly/update/macos/releases.json - --endpoint-url $R2_URL --region auto | jq -r .currentRelease)
+                  echo "version=$(scripts/generate-nightly-version.ts --latest $LATEST)" >> $GITHUB_OUTPUT
+              env:
+                  AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+                  AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_TOKEN }}
+                  R2_URL: ${{ secrets.CF_R2_S3_API }}

.github/workflows/packages_index.yaml

@@ -5,14 +5,20 @@ on:
         branches: [develop]
         paths:
             - "packages.element.io/**"
-    # Trigger a daily rebuild for nightlies
+    # Trigger a daily rebuild for (mac-mini built) Nightly builds
     schedule:
         - cron: "0 11 * * *"
+  # Trigger after Nightly builds are deployed
+    workflow_run:
+        workflows: [ "Build and Deploy" ]
+        types:
+            - completed
     # Manual trigger for rebuilding for releases
     workflow_dispatch: {}
 jobs:
     deploy:
         name: "Deploy"
+        if: github.event != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
         runs-on: ubuntu-latest
         environment: packages.element.io
         env: