From 7d1a8fb75371f73db8b525a8b5701a489786b803 Mon Sep 17 00:00:00 2001 From: Zheng Feng Date: Sat, 11 Jul 2026 18:29:48 +0800 Subject: [PATCH] ci: add nightly build for images + helm chart (#4050) * ci: add nightly build workflow for images + helm chart Nightly build of backend/frontend/provisioner images and the helm chart, pushed to GHCR with nightly + nightly-YYYYMMDD tags (latest stays on v* releases). amd64 only. Gated to the upstream repo (bytedance/deer-flow). Documented in RELEASING.md. * ci(nightly): harden chart patch + dedupe config_version check Address PR #4050 review feedback: - Gate the in-workflow chart patches with grep assertions so a drifted Chart.yaml/values.yaml fails loudly instead of silently shipping a chart that pulls the release `latest` images (sed exits 0 on zero matches). - Suffix the nightly chart version with the short SHA (-nightly.-) so a same-day re-dispatch re-publishes cleanly; OCI chart versions are immutable and otherwise collide. - Note in the image-tags comment that :nightly- is mutable within a day and :sha- is the only truly immutable pin. - Extract the config_version drift check into scripts/check_config_version.sh, shared by chart.yaml and nightly.yaml, so the parsing logic lives in one place. --- .github/workflows/chart.yaml | 19 +-- .github/workflows/nightly.yaml | 203 ++++++++++++++++++++++++++++++++ RELEASING.md | 30 +++++ scripts/check_config_version.sh | 50 ++++++++ 4 files changed, 287 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/nightly.yaml create mode 100755 scripts/check_config_version.sh diff --git a/.github/workflows/chart.yaml b/.github/workflows/chart.yaml index 4240e201b..5d7a99d34 100644 --- a/.github/workflows/chart.yaml +++ b/.github/workflows/chart.yaml @@ -20,6 +20,7 @@ on: - "deploy/helm/deer-flow/**" - "config.example.yaml" - ".github/workflows/chart.yaml" + - "scripts/check_config_version.sh" jobs: validate-chart: @@ -47,22 +48,10 @@ jobs: # never warns) but means the chart's config is authored against an older # schema. Bump it in values.yaml and the README example. config_version # gates no runtime behavior - it only drives the outdated-warning - so a - # bare version bump needs no field changes. + # bare version bump needs no field changes. Logic lives in + # scripts/check_config_version.sh (shared with nightly.yaml). - name: config_version drift check - run: | - set -eu - example=$(grep -E '^config_version:[[:space:]]+[0-9]+' config.example.yaml | head -1 | awk '{print $2}') - chart=$(awk '/^config:[[:space:]]*\|/{f=1; next} f && /^[[:space:]]+config_version:[[:space:]]+[0-9]+/ {print $2; exit}' deploy/helm/deer-flow/values.yaml) - echo "config.example.yaml config_version=$example" - echo "chart values.yaml config_version=$chart" - if [ -z "$example" ] || [ -z "$chart" ]; then - echo "::error::could not parse config_version from one of the files" - exit 1 - fi - if [ "$chart" -lt "$example" ]; then - echo "::error::chart config_version ($chart) is behind config.example.yaml ($example). Bump 'config_version' in deploy/helm/deer-flow/values.yaml (and the README example) to $example." - exit 1 - fi + run: bash scripts/check_config_version.sh verify-versions: # Gate the release: every version source must match the v* tag. A forgotten diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml new file mode 100644 index 000000000..3eef3018f --- /dev/null +++ b/.github/workflows/nightly.yaml @@ -0,0 +1,203 @@ +name: Nightly Build + +# Nightly build of the three DeerFlow container images (backend, frontend, +# provisioner) and the Helm chart, published to GHCR from the default branch. +# +# This mirrors the tag-driven release workflows (container.yaml + chart.yaml) +# but trades the `v*` tag for date-based tags, since nightly ships unreleased +# `main`. The `verify-versions` gate is intentionally skipped - there is no tag +# to match - and `latest` is left untouched so it keeps tracking the last `v*` +# release. Images are amd64-only to match the release builds. +# +# Artifacts (under the running repo's owner): +# ghcr.io//deer-flow-{backend,frontend,provisioner}:nightly +# ghcr.io//deer-flow-{backend,frontend,provisioner}:nightly-YYYYMMDD +# oci://ghcr.io//deer-flow chart version -nightly.YYYYMMDD- +# +# The nightly chart defaults image.tag=nightly and image.registry=ghcr.io/ +# (patched in-workflow, never committed), so installing it pulls the matching +# nightly images with no values overrides. +# +# Restricted to the upstream repo: every job is gated on +# `github.repository == 'bytedance/deer-flow'`, so a scheduled run or manual +# dispatch on a fork skips all jobs rather than pushing to the fork's own GHCR +# namespace. Scheduled workflows also only fire on the default branch. + +on: + schedule: + - cron: "0 16 * * *" # 16:00 UTC daily (adjust as needed) + workflow_dispatch: + +concurrency: + group: nightly + # Don't cancel a running nightly - a mid-cancel leaves a half-pushed set. + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + +jobs: + prepare: + if: github.repository == 'bytedance/deer-flow' + runs-on: ubuntu-latest + outputs: + date: ${{ steps.date.outputs.date }} + steps: + - name: Set nightly date (UTC) + id: date + # Shared by build-images (image tag) and publish-chart (chart version) + # so the two stay in sync for a given run. + run: echo "date=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + + validate-chart: + if: github.repository == 'bytedance/deer-flow' + # Catch a broken render or a stale config_version before publish. A broken + # chart published under an OCI version is immutable (GHCR won't let you + # overwrite --version), so a regression must fail here, not on install. + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3 + + - name: Lint chart + run: helm lint deploy/helm/deer-flow + - name: Validate templates render + run: helm template deer-flow deploy/helm/deer-flow --include-crds >/dev/null + + # The chart's `config:` block embeds a config_version that must not fall + # behind config.example.yaml. Shared with chart.yaml via + # scripts/check_config_version.sh so the two workflows can't drift. + - name: config_version drift check + run: bash scripts/check_config_version.sh + + build-images: + needs: prepare + if: github.repository == 'bytedance/deer-flow' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + strategy: + fail-fast: false + matrix: + include: + - component: backend + context: . + file: backend/Dockerfile + # Bake the `postgres` extra so multi-replica (K8s/Helm) deployments + # can use shared Postgres. Matches container.yaml's release image. + build-args: "UV_EXTRAS=postgres" + - component: frontend + context: . + file: frontend/Dockerfile + build-args: "" + - component: provisioner + context: docker/provisioner + file: docker/provisioner/Dockerfile + build-args: "" + env: + IMAGE_NAME: ${{ github.repository }}-${{ matrix.component }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3 + - name: Log in to the Container registry + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # `nightly` rolls forward each run; `nightly-YYYYMMDD` is pinned to a + # day but mutable within it (a same-day re-dispatch overwrites it); + # `sha-` is the only truly immutable tag. No `latest` (that + # stays on v* releases) and no branch/tag refs. + tags: | + type=raw,value=nightly + type=raw,value=nightly-${{ needs.prepare.outputs.date }} + type=sha + - name: Build and push Docker image + id: push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: ${{ matrix.build-args }} + - name: Generate artifact attestation + uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be #v2.4.0 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + + publish-chart: + # Ship the chart only after images build - it references all three, so a + # failed image build withholds the chart rather than publishing one that + # would fail to pull in-cluster. + needs: [prepare, validate-chart, build-images] + if: github.repository == 'bytedance/deer-flow' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + DATE: ${{ needs.prepare.outputs.date }} + OWNER: ${{ github.repository_owner }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 #v6.0.3 + + - name: Patch chart to a nightly version + nightly image defaults + # Bumps Chart.yaml version/appVersion to -nightly.- + # (valid semver prerelease; the short SHA makes each dispatch's version + # unique, so a same-day re-dispatch re-publishes cleanly - OCI chart + # versions are immutable and otherwise can't be overwritten). Repoints + # the chart's default image registry/tag at the nightly build. Patches + # are in-workflow only - nothing is committed back. + run: | + set -eu + CHART=deploy/helm/deer-flow + BASE=$(grep -m1 '^version:' "$CHART/Chart.yaml" | awk '{print $2}') + SHORT_SHA="${GITHUB_SHA::7}" + NIGHTLY="${BASE}-nightly.${DATE}-${SHORT_SHA}" + echo "Nightly chart version: ${NIGHTLY}" + sed -i "s|^version:.*|version: ${NIGHTLY}|" "$CHART/Chart.yaml" + sed -i "s|^appVersion:.*|appVersion: \"${NIGHTLY}\"|" "$CHART/Chart.yaml" + sed -i "s|^ registry: \"\".*| registry: \"ghcr.io/${OWNER}\"|" "$CHART/values.yaml" + sed -i 's|^ tag: "latest"| tag: "nightly"|' "$CHART/values.yaml" + # Gate the patches: sed exits 0 even on zero matches, so a drifted + # Chart.yaml/values.yaml would otherwise ship a chart that silently + # pulls the wrong (release `latest`) images. Fail loudly if any patch + # missed. + grep -q "^version: ${NIGHTLY}$" "$CHART/Chart.yaml" || { echo "::error::Chart.yaml version sed did not apply"; exit 1; } + grep -q "^appVersion: \"${NIGHTLY}\"$" "$CHART/Chart.yaml" || { echo "::error::Chart.yaml appVersion sed did not apply"; exit 1; } + grep -q '^ registry: "ghcr.io/' "$CHART/values.yaml" || { echo "::error::values.yaml registry sed did not apply"; exit 1; } + grep -q '^ tag: "nightly"' "$CHART/values.yaml" || { echo "::error::values.yaml tag sed did not apply"; exit 1; } + echo "--- Chart.yaml (head) ---"; sed -n '1,7p' "$CHART/Chart.yaml" + echo "--- image block ---"; sed -n '11,14p' "$CHART/values.yaml" + + - name: Lint patched chart + run: helm lint deploy/helm/deer-flow + + - name: Log in to GHCR + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | \ + helm registry login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Package and push chart + run: | + helm package deploy/helm/deer-flow --destination ./packages + for pkg in ./packages/*.tgz; do + echo "--- pushing $pkg" + helm push "$pkg" oci://ghcr.io/${{ github.repository_owner }} + done diff --git a/RELEASING.md b/RELEASING.md index 5b866e630..60b335cd2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -74,6 +74,36 @@ tag, the release is blocked (see [Version gate](#version-gate)). helm install deer-flow oci://ghcr.io//deer-flow --version 2.2.0 ``` +## Nightly builds + +`.github/workflows/nightly.yaml` runs on a schedule (and `workflow_dispatch`) +to publish the same three images plus the chart from unreleased `main`. It is +**not** gated by the version check (there is no `v*` tag) and it does **not** +touch the `latest` tag, which stays pinned to the last `v*` release. Every job +is gated on `github.repository == 'bytedance/deer-flow'`, so it only runs on +the upstream repo - a scheduled run or manual dispatch on a fork skips all jobs. + +Artifacts (under the running repo's owner, where `` is `YYYYMMDD`): + +- Images: `ghcr.io//deer-flow-{backend,frontend,provisioner}:nightly` + (rolling, overwritten each run) and `:nightly-` (pinned to a day, but + mutable within it - a same-day re-dispatch overwrites it). For a truly + immutable pin, use `:sha-`. +- Chart: `oci://ghcr.io//deer-flow`, version `-nightly.-` + (e.g. `2.2.0-nightly.20260710-77a3652`). The short SHA makes each dispatch's + chart version unique, so a same-day re-dispatch re-publishes cleanly (OCI + chart versions are immutable and otherwise can't be overwritten). The + packaged chart defaults `image.registry=ghcr.io/` and + `image.tag=nightly`, so installing it pulls the matching nightly images with + no values overrides: + ```bash + helm install deer-flow oci://ghcr.io//deer-flow \ + --version 2.2.0-nightly.20260710-77a3652 + ``` + +The chart version is patched in-workflow only - `Chart.yaml` and `values.yaml` +in the repo are never modified. + ## Version gate Both publishing workflows call `.github/workflows/verify-versions.yml` as their diff --git a/scripts/check_config_version.sh b/scripts/check_config_version.sh new file mode 100755 index 000000000..035ca9598 --- /dev/null +++ b/scripts/check_config_version.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Check the Helm chart's embedded config_version is not behind config.example.yaml. +# +# The chart's `config:` block in deploy/helm/deer-flow/values.yaml embeds a +# config_version that must not lag config.example.yaml. A stale version is +# silent in-cluster (the image ships no example to compare against, so +# _check_config_version never warns) but means the chart's config is authored +# against an older schema - so this fails the build, not a user's install. +# config_version gates no runtime behavior; it only drives the outdated-warning, +# so a bare version bump needs no field changes. +# +# Usage: +# scripts/check_config_version.sh +# +# Called by .github/workflows/chart.yaml (validate-chart, on PRs + v* tags) and +# .github/workflows/nightly.yaml (validate-chart) so both stay in sync. +# +# Exit status is 0 when the chart is current, 1 otherwise. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +EXAMPLE_YAML="$ROOT/config.example.yaml" +VALUES_YAML="$ROOT/deploy/helm/deer-flow/values.yaml" + +for f in "$EXAMPLE_YAML" "$VALUES_YAML"; do + if [ ! -f "$f" ]; then + echo "::error::missing file: $f" >&2 + exit 1 + fi +done + +example=$(grep -E '^config_version:[[:space:]]+[0-9]+' "$EXAMPLE_YAML" | head -1 | awk '{print $2}') +chart=$(awk '/^config:[[:space:]]*\|/{f=1; next} f && /^[[:space:]]+config_version:[[:space:]]+[0-9]+/ {print $2; exit}' "$VALUES_YAML") + +printf 'config.example.yaml config_version=%s\n' "$example" +printf 'chart values.yaml config_version=%s\n' "$chart" + +if [ -z "$example" ] || [ -z "$chart" ]; then + echo "::error::could not parse config_version from one of the files" >&2 + exit 1 +fi + +if [ "$chart" -lt "$example" ]; then + echo "::error::chart config_version ($chart) is behind config.example.yaml ($example). Bump 'config_version' in deploy/helm/deer-flow/values.yaml (and the README example) to $example." >&2 + exit 1 +fi + +echo "OK - chart config_version ($chart) is current with config.example.yaml ($example)."