ci: add nightly build for images + helm chart (#4050)

* ci: add nightly build workflow for images + helm chart Nightly build of backend/frontend/provisioner images and the helm chart, pushed to GHCR with nightly + nightly-YYYYMMDD tags (latest stays on v* releases). amd64 only. Gated to the upstream repo (bytedance/deer-flow). Documented in RELEASING.md. * ci(nightly): harden chart patch + dedupe config_version check Address PR #4050 review feedback: - Gate the in-workflow chart patches with grep assertions so a drifted Chart.yaml/values.yaml fails loudly instead of silently shipping a chart that pulls the release `latest` images (sed exits 0 on zero matches). - Suffix the nightly chart version with the short SHA (<base>-nightly.<date>-<sha>) so a same-day re-dispatch re-publishes cleanly; OCI chart versions are immutable and otherwise collide. - Note in the image-tags comment that :nightly-<date> is mutable within a day and :sha-<short> is the only truly immutable pin. - Extract the config_version drift check into scripts/check_config_version.sh, shared by chart.yaml and nightly.yaml, so the parsing logic lives in one place.
2026-07-21 02:05:45 +00:00 · 2026-07-11 18:29:48 +08:00
parent 3bc3af2530
commit 7d1a8fb753
4 changed files with 287 additions and 15 deletions
@@ -20,6 +20,7 @@ on:
      - "deploy/helm/deer-flow/**"
      - "config.example.yaml"
      - ".github/workflows/chart.yaml"
+      - "scripts/check_config_version.sh"

 jobs:
  validate-chart:
@@ -47,22 +48,10 @@ jobs:
      # never warns) but means the chart's config is authored against an older
      # schema. Bump it in values.yaml and the README example. config_version
      # gates no runtime behavior - it only drives the outdated-warning - so a
-      # bare version bump needs no field changes.
+      # bare version bump needs no field changes. Logic lives in
+      # scripts/check_config_version.sh (shared with nightly.yaml).
      - name: config_version drift check
-        run: |
-          set -eu
-          example=$(grep -E '^config_version:[[:space:]]+[0-9]+' config.example.yaml | head -1 | awk '{print $2}')
-          chart=$(awk '/^config:[[:space:]]*\|/{f=1; next} f && /^[[:space:]]+config_version:[[:space:]]+[0-9]+/ {print $2; exit}' deploy/helm/deer-flow/values.yaml)
-          echo "config.example.yaml config_version=$example"
-          echo "chart values.yaml     config_version=$chart"
-          if [ -z "$example" ] || [ -z "$chart" ]; then
-            echo "::error::could not parse config_version from one of the files"
-            exit 1
-          fi
-          if [ "$chart" -lt "$example" ]; then
-            echo "::error::chart config_version ($chart) is behind config.example.yaml ($example). Bump 'config_version' in deploy/helm/deer-flow/values.yaml (and the README example) to $example."
-            exit 1
-          fi
+        run: bash scripts/check_config_version.sh

  verify-versions:
    # Gate the release: every version source must match the v* tag. A forgotten
@@ -0,0 +1,203 @@
+name: Nightly Build
+
+# Nightly build of the three DeerFlow container images (backend, frontend,
+# provisioner) and the Helm chart, published to GHCR from the default branch.
+#
+# This mirrors the tag-driven release workflows (container.yaml + chart.yaml)
+# but trades the `v*` tag for date-based tags, since nightly ships unreleased
+# `main`. The `verify-versions` gate is intentionally skipped - there is no tag
+# to match - and `latest` is left untouched so it keeps tracking the last `v*`
+# release. Images are amd64-only to match the release builds.
+#
+# Artifacts (under the running repo's owner):
+#   ghcr.io/<owner>/deer-flow-{backend,frontend,provisioner}:nightly
+#   ghcr.io/<owner>/deer-flow-{backend,frontend,provisioner}:nightly-YYYYMMDD
+#   oci://ghcr.io/<owner>/deer-flow   chart version <base>-nightly.YYYYMMDD-<sha>
+#
+# The nightly chart defaults image.tag=nightly and image.registry=ghcr.io/<owner>
+# (patched in-workflow, never committed), so installing it pulls the matching
+# nightly images with no values overrides.
+#
+# Restricted to the upstream repo: every job is gated on
+# `github.repository == 'bytedance/deer-flow'`, so a scheduled run or manual
+# dispatch on a fork skips all jobs rather than pushing to the fork's own GHCR
+# namespace. Scheduled workflows also only fire on the default branch.
+
+on:
+  schedule:
+    - cron: "0 16 * * *" # 16:00 UTC daily (adjust as needed)
+  workflow_dispatch:
+
+concurrency:
+  group: nightly
+  # Don't cancel a running nightly - a mid-cancel leaves a half-pushed set.
+  cancel-in-progress: false
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  prepare:
+    if: github.repository == 'bytedance/deer-flow'
+    runs-on: ubuntu-latest
+    outputs:
+      date: ${{ steps.date.outputs.date }}
+    steps:
+      - name: Set nightly date (UTC)
+        id: date
+        # Shared by build-images (image tag) and publish-chart (chart version)
+        # so the two stay in sync for a given run.
+        run: echo "date=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT"
+
+  validate-chart:
+    if: github.repository == 'bytedance/deer-flow'
+    # Catch a broken render or a stale config_version before publish. A broken
+    # chart published under an OCI version is immutable (GHCR won't let you
+    # overwrite --version), so a regression must fail here, not on install.
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3
+
+      - name: Lint chart
+        run: helm lint deploy/helm/deer-flow
+      - name: Validate templates render
+        run: helm template deer-flow deploy/helm/deer-flow --include-crds >/dev/null
+
+      # The chart's `config:` block embeds a config_version that must not fall
+      # behind config.example.yaml. Shared with chart.yaml via
+      # scripts/check_config_version.sh so the two workflows can't drift.
+      - name: config_version drift check
+        run: bash scripts/check_config_version.sh
+
+  build-images:
+    needs: prepare
+    if: github.repository == 'bytedance/deer-flow'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - component: backend
+            context: .
+            file: backend/Dockerfile
+            # Bake the `postgres` extra so multi-replica (K8s/Helm) deployments
+            # can use shared Postgres. Matches container.yaml's release image.
+            build-args: "UV_EXTRAS=postgres"
+          - component: frontend
+            context: .
+            file: frontend/Dockerfile
+            build-args: ""
+          - component: provisioner
+            context: docker/provisioner
+            file: docker/provisioner/Dockerfile
+            build-args: ""
+    env:
+      IMAGE_NAME: ${{ github.repository }}-${{ matrix.component }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3
+      - name: Log in to the Container registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          # `nightly` rolls forward each run; `nightly-YYYYMMDD` is pinned to a
+          # day but mutable within it (a same-day re-dispatch overwrites it);
+          # `sha-<short>` is the only truly immutable tag. No `latest` (that
+          # stays on v* releases) and no branch/tag refs.
+          tags: |
+            type=raw,value=nightly
+            type=raw,value=nightly-${{ needs.prepare.outputs.date }}
+            type=sha
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.file }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: ${{ matrix.build-args }}
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be #v2.4.0
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+
+  publish-chart:
+    # Ship the chart only after images build - it references all three, so a
+    # failed image build withholds the chart rather than publishing one that
+    # would fail to pull in-cluster.
+    needs: [prepare, validate-chart, build-images]
+    if: github.repository == 'bytedance/deer-flow'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    env:
+      DATE: ${{ needs.prepare.outputs.date }}
+      OWNER: ${{ github.repository_owner }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3
+
+      - name: Patch chart to a nightly version + nightly image defaults
+        # Bumps Chart.yaml version/appVersion to <base>-nightly.<date>-<sha>
+        # (valid semver prerelease; the short SHA makes each dispatch's version
+        # unique, so a same-day re-dispatch re-publishes cleanly - OCI chart
+        # versions are immutable and otherwise can't be overwritten). Repoints
+        # the chart's default image registry/tag at the nightly build. Patches
+        # are in-workflow only - nothing is committed back.
+        run: |
+          set -eu
+          CHART=deploy/helm/deer-flow
+          BASE=$(grep -m1 '^version:' "$CHART/Chart.yaml" | awk '{print $2}')
+          SHORT_SHA="${GITHUB_SHA::7}"
+          NIGHTLY="${BASE}-nightly.${DATE}-${SHORT_SHA}"
+          echo "Nightly chart version: ${NIGHTLY}"
+          sed -i "s|^version:.*|version: ${NIGHTLY}|" "$CHART/Chart.yaml"
+          sed -i "s|^appVersion:.*|appVersion: \"${NIGHTLY}\"|" "$CHART/Chart.yaml"
+          sed -i "s|^  registry: \"\".*|  registry: \"ghcr.io/${OWNER}\"|" "$CHART/values.yaml"
+          sed -i 's|^  tag: "latest"|  tag: "nightly"|' "$CHART/values.yaml"
+          # Gate the patches: sed exits 0 even on zero matches, so a drifted
+          # Chart.yaml/values.yaml would otherwise ship a chart that silently
+          # pulls the wrong (release `latest`) images. Fail loudly if any patch
+          # missed.
+          grep -q "^version: ${NIGHTLY}$"         "$CHART/Chart.yaml"  || { echo "::error::Chart.yaml version sed did not apply"; exit 1; }
+          grep -q "^appVersion: \"${NIGHTLY}\"$"   "$CHART/Chart.yaml"  || { echo "::error::Chart.yaml appVersion sed did not apply"; exit 1; }
+          grep -q '^  registry: "ghcr.io/'         "$CHART/values.yaml" || { echo "::error::values.yaml registry sed did not apply"; exit 1; }
+          grep -q '^  tag: "nightly"'              "$CHART/values.yaml" || { echo "::error::values.yaml tag sed did not apply"; exit 1; }
+          echo "--- Chart.yaml (head) ---"; sed -n '1,7p' "$CHART/Chart.yaml"
+          echo "--- image block ---"; sed -n '11,14p' "$CHART/values.yaml"
+
+      - name: Lint patched chart
+        run: helm lint deploy/helm/deer-flow
+
+      - name: Log in to GHCR
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | \
+            helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Package and push chart
+        run: |
+          helm package deploy/helm/deer-flow --destination ./packages
+          for pkg in ./packages/*.tgz; do
+            echo "--- pushing $pkg"
+            helm push "$pkg" oci://ghcr.io/${{ github.repository_owner }}
+          done
@@ -74,6 +74,36 @@ tag, the release is blocked (see [Version gate](#version-gate)).
  helm install deer-flow oci://ghcr.io/<owner>/deer-flow --version 2.2.0
  ```

+## Nightly builds
+
+`.github/workflows/nightly.yaml` runs on a schedule (and `workflow_dispatch`)
+to publish the same three images plus the chart from unreleased `main`. It is
+**not** gated by the version check (there is no `v*` tag) and it does **not**
+touch the `latest` tag, which stays pinned to the last `v*` release. Every job
+is gated on `github.repository == 'bytedance/deer-flow'`, so it only runs on
+the upstream repo - a scheduled run or manual dispatch on a fork skips all jobs.
+
+Artifacts (under the running repo's owner, where `<date>` is `YYYYMMDD`):
+
+- Images: `ghcr.io/<owner>/deer-flow-{backend,frontend,provisioner}:nightly`
+  (rolling, overwritten each run) and `:nightly-<date>` (pinned to a day, but
+  mutable within it - a same-day re-dispatch overwrites it). For a truly
+  immutable pin, use `:sha-<short>`.
+- Chart: `oci://ghcr.io/<owner>/deer-flow`, version `<base>-nightly.<date>-<sha>`
+  (e.g. `2.2.0-nightly.20260710-77a3652`). The short SHA makes each dispatch's
+  chart version unique, so a same-day re-dispatch re-publishes cleanly (OCI
+  chart versions are immutable and otherwise can't be overwritten). The
+  packaged chart defaults `image.registry=ghcr.io/<owner>` and
+  `image.tag=nightly`, so installing it pulls the matching nightly images with
+  no values overrides:
+  ```bash
+  helm install deer-flow oci://ghcr.io/<owner>/deer-flow \
+    --version 2.2.0-nightly.20260710-77a3652
+  ```
+
+The chart version is patched in-workflow only - `Chart.yaml` and `values.yaml`
+in the repo are never modified.
+
 ## Version gate

 Both publishing workflows call `.github/workflows/verify-versions.yml` as their
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Check the Helm chart's embedded config_version is not behind config.example.yaml.
+#
+# The chart's `config:` block in deploy/helm/deer-flow/values.yaml embeds a
+# config_version that must not lag config.example.yaml. A stale version is
+# silent in-cluster (the image ships no example to compare against, so
+# _check_config_version never warns) but means the chart's config is authored
+# against an older schema - so this fails the build, not a user's install.
+# config_version gates no runtime behavior; it only drives the outdated-warning,
+# so a bare version bump needs no field changes.
+#
+# Usage:
+#   scripts/check_config_version.sh
+#
+# Called by .github/workflows/chart.yaml (validate-chart, on PRs + v* tags) and
+# .github/workflows/nightly.yaml (validate-chart) so both stay in sync.
+#
+# Exit status is 0 when the chart is current, 1 otherwise.
+
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+EXAMPLE_YAML="$ROOT/config.example.yaml"
+VALUES_YAML="$ROOT/deploy/helm/deer-flow/values.yaml"
+
+for f in "$EXAMPLE_YAML" "$VALUES_YAML"; do
+  if [ ! -f "$f" ]; then
+    echo "::error::missing file: $f" >&2
+    exit 1
+  fi
+done
+
+example=$(grep -E '^config_version:[[:space:]]+[0-9]+' "$EXAMPLE_YAML" | head -1 | awk '{print $2}')
+chart=$(awk '/^config:[[:space:]]*\|/{f=1; next} f && /^[[:space:]]+config_version:[[:space:]]+[0-9]+/ {print $2; exit}' "$VALUES_YAML")
+
+printf 'config.example.yaml config_version=%s\n' "$example"
+printf 'chart values.yaml     config_version=%s\n' "$chart"
+
+if [ -z "$example" ] || [ -z "$chart" ]; then
+  echo "::error::could not parse config_version from one of the files" >&2
+  exit 1
+fi
+
+if [ "$chart" -lt "$example" ]; then
+  echo "::error::chart config_version ($chart) is behind config.example.yaml ($example). Bump 'config_version' in deploy/helm/deer-flow/values.yaml (and the README example) to $example." >&2
+  exit 1
+fi
+
+echo "OK - chart config_version ($chart) is current with config.example.yaml ($example)."