# bp-self-sovereign-cutover — Catalyst bootstrap-kit Blueprint, slot 06a.
#
# Post-handover Self-Sovereignty Cutover. Installs DORMANT — see chart
# README + ADR-0002. Eight step PodSpec ConfigMaps + the registry-pivot
# DaemonSet land on the new Sovereign at HelmRelease apply time; the
# catalyst-api cutover endpoint (issue #792) reads them by label
# selector and stamps Jobs only on operator action.
#
# Slot 06a sits between the existing post-handover slots in the
# bootstrap-kit ordering. It depends on bp-gitea + bp-harbor so the
# step ConfigMaps reference real, healthy local Gitea + Harbor
# Services at trigger time.
#
# Wrapper chart: platform/self-sovereign-cutover/chart/
# Reconciled by: Flux on the new Sovereign's k3s control plane.
#
# Why disableWait: true
#   The chart is dormant — no Job is created at install time. The only
#   workload that actually runs is the registry-pivot DaemonSet, which
#   never converges on its own (it waits for the cutover-status
#   ConfigMap to flip registriesYamlActive=v2). disableWait: true makes
#   Helm exit when the manifests apply rather than waiting on a Ready
#   condition that never fires.

---
apiVersion: v1
kind: Namespace
metadata:
  name: catalyst
  labels:
    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
---
apiVersion: source.toolkit.fluxcd.io/v1beta2
kind: HelmRepository
metadata:
  name: bp-self-sovereign-cutover
  namespace: flux-system
spec:
  type: oci
  interval: 15m
  # Pre-cutover (Phase-1) — sources from openova-io GHCR, identical to
  # every other bootstrap-kit slot. The cutover itself (step 06,
  # helmrepository-patches) is what flips this URL to the local Harbor
  # post-handover; until then this Sovereign is soft-tethered like the
  # rest of the kit.
  url: oci://ghcr.io/openova-io
  secretRef:
    name: ghcr-pull
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: bp-self-sovereign-cutover
  namespace: flux-system
  labels:
    # slot drives the openova-flow adapter's Phase derivation
    # (clusters/_template/bootstrap-kit/<NN>-...yaml encodes the
    # install order). component=cutover overrides the slot rule so
    # this HR lands in Phase 2 (Cutover) on the canvas, NOT Phase 1.
    catalyst.openova.io/slot: "06a"
    catalyst.openova.io/component: cutover
spec:
  interval: 15m
  releaseName: self-sovereign-cutover
  # targetNamespace = catalyst because the catalyst-api cutover endpoint
  # defaults its discovery namespace to "catalyst"
  # (CATALYST_CUTOVER_NAMESPACE in
  # products/catalyst/bootstrap/api/internal/handler/cutover.go). Keeping
  # the chart resources colocated with catalyst-api avoids a cross-
  # namespace selector + a CATALYST_CUTOVER_NAMESPACE env override on
  # every Sovereign.
  targetNamespace: catalyst
  dependsOn:
    # Both bp-gitea and bp-harbor must be Ready before the chart's
    # PodSpec ConfigMaps reference real Services. The chart itself is
    # dormant so an early apply is benign — but operator workflow
    # ordering puts cutover after these two come up.
    - name: bp-gitea
    - name: bp-harbor
    # NB on issue #1871 (TBD-A24 cutover↔gateway circular deadlock):
    # PR #1875 initially added `- name: sovereign-tls` to this list.
    # That fix was UNRESOLVABLE in Flux: HelmRelease.dependsOn can
    # only reference other HelmReleases (helm.toolkit.fluxcd.io/v2),
    # but `sovereign-tls` is a Flux Kustomization. helm-controller
    # logged `helmreleases.helm.toolkit.fluxcd.io "sovereign-tls" not
    # found` on t27 fresh-prov 2026-05-18, and bp-self-sovereign-
    # cutover sat forever in dependency-wait — cutover never fired,
    # handover never fired (A84 empirical test). The dependsOn entry
    # was reverted in chart 0.1.32; the real fix moved INTO the
    # chart's Step-06 helmrepository-patches Job, which now waits for
    # `gateway.networking.k8s.io/cilium-gateway` in `kube-system` to
    # report `Programmed=True` BEFORE rewriting any HelmRepository
    # URL. That ordering breaks the deadlock without needing a cross-
    # kind dependsOn. See platform/self-sovereign-cutover/chart/
    # templates/06-helmrepository-patches-job.yaml (Phase -1 gateway-
    # wait block) for the implementation.
  chart:
    spec:
      chart: bp-self-sovereign-cutover
      # 0.1.1: Step-1 gitea-mirror script uses BusyBox-wget-compatible
      # Authorization: Basic <b64> header instead of --user/--password
      # which alpine/git's BusyBox wget does not support.
      # 0.1.2: Step-1 explicitly creates the Gitea org BEFORE the repo —
      # POST /orgs/<org>/repos returns 404 if the org is missing, the
      # /user/repos fallback would create under gitea_admin (wrong path
      # for the subsequent push). Caught live on otech103 2026-05-04.
      # 0.1.3: replace `git push --mirror` with `git push --all + --tags`
      # so Gitea's hooks don't decline GitHub-specific refs/pull/<n>
      # refs (which --mirror would try to push). Branches+tags are what
      # Flux GitRepository needs; PR refs are upstream-only metadata.
      # 0.1.4: Step-1 uses `git clone --bare` (not --mirror) + explicit
      # refspec push of refs/heads/* and refs/tags/* only. --all in a
      # mirror clone still pushed refs/pull/* — caught live otech103.
      # 0.1.5: harborInternalURL fix — bp-harbor service is `harbor-core`
      # not `harbor-harbor-core` (release name doesn't double-prefix).
      # Caught live otech103 — Step-2 curl exit 6 (couldn't resolve).
      # 0.1.6: proxy-ghcr registryType "github" → "github-ghcr" (the
      # canonical Harbor adapter name for GHCR proxy-cache, per Harbor
      # 2.x docs). Caught live otech103 — Harbor 500 "adapter factory
      # for github not found".
      # 0.1.7: proxy-quay registryType "quay" → "docker-registry" —
      # Harbor's "quay" adapter rejects project metadata.proxy_cache
      # with HTTP 400. Quay speaks plain v2 so generic docker-registry
      # is correct. Caught live otech103 — 4/7 proxy-cache projects
      # were created OK, blocked at proxy-quay.
      # 0.1.8: bitnami/kubectl tag :1.31 → :1.31.4 (bitnami doesn't tag
      # at minor-version, only patch). Caught live otech103 — Step-5
      # Pod hit DeadlineExceeded after 10m of ImagePullBackOff for
      # docker.io/bitnami/kubectl:1.31 (404 not found).
      # 0.1.9: bitnami/kubectl :1.31.4 ALSO 404 (Bitnami deprecated
      # public Docker Hub in 2025). Switched to alpine/k8s:1.31.4 —
      # canonical alpine-based image with kubectl + helm + k8s CLI
      # surface, actively maintained.
      # 0.1.10: catalystAPI.namespace `catalyst-platform` → `catalyst-
      # system` (the actual Sovereign-side namespace). Caught live
      # otech103 — Step-7 `deployment catalyst-api not found`.
      # 0.1.11: Step-8 egress-block-test pivoted from CiliumNetworkPolicy
      # (egressDeny + toFQDNs unsupported in Cilium 1.16) to a passive
      # architectural-state assertion + ${durationSeconds}s survival
      # window. Same proof shape, valid Cilium policy. Caught live
      # otech103 — strict-decoding error 'unknown field toFQDNs'.
      # 0.1.12: Step-8 verification tolerates slot-managed self-ref
      # HelmRepositories (bp-newapi + bp-self-sovereign-cutover) which
      # Flux Kustomization re-applies from bootstrap-kit slots after
      # Step-6's patch. Data-plane impact null — they're not pulled
      # again until next cutover cycle. Caught live otech103.
      # 0.1.13: Step-8 survival window captures BASELINE NotReady set
      # before entering the window, then only fails on NEW Ready=False
      # transitions (regressions). Pre-existing failures (Crossplane
      # provider CRD ordering, etc.) don't poison the sovereignty
      # verdict — sovereignty asks "did cutover break anything", not
      # "is the cluster perfect". Caught live otech103 — infrastructure
      # -config Kustomization had been NotReady for 4h pre-cutover.
      # 0.1.14: Step-1 gitea-mirror replaces one-shot create+push with
      # Gitea native /repos/migrate `mirror=true` + mirror_interval=10m
      # so the local Gitea polls upstream GitHub on a recurring 10-min
      # interval. Closes the "Sovereign drifts from upstream main
      # forever after Day-2 cutover" bug — hit twice on otech103
      # 2026-05-04 requiring manual `git fetch` per chart rollout. (#870)
      # 0.1.16: Auto-trigger via Helm post-install Job (#933). Handover
      # is not "done" until cutover has run; the operator must NOT have
      # to click a CTA. New `trigger.auto: true` (default) fires a
      # post-install Job that POSTs /api/v1/sovereign/cutover/start
      # on catalyst-api after the step ConfigMaps land. catalyst-api
      # handles idempotency via the durable status ConfigMap, so the
      # hook is safe on every install + every upgrade. Coupled with the
      # cutoverStatusResponse.State field fix on the API side which
      # closes the otech113 `invalid CutoverState: <undefined>` bug.
      # 0.1.17: Two-bug fix surfaced live on otech113 2026-05-05 (#935):
      #   Bug 1 — Step 02 (harbor-projects) Job in `catalyst` ns was
      #     hitting `secret "harbor-core" not found` because the
      #     upstream Harbor `harbor-core` Secret only exists in
      #     `harbor` ns and K8s forbids cross-namespace secretKeyRef.
      #     Fix lives in bp-harbor 1.2.14: a Catalyst-curated
      #     `harbor-admin` Secret is now emitted in the harbor ns
      #     with Reflector annotations mirroring it into `catalyst`
      #     so the Job's secretKeyRef resolves automatically. This
      #     chart's values.yaml `harbor.adminSecretRef.name` is now
      #     `harbor-admin` (was `harbor-core`).
      #   Bug 2 — 0.1.16 auto-trigger Job POSTed
      #     /api/v1/sovereign/cutover/start which lives behind
      #     RequireSession middleware → 401 forever (no session
      #     cookie on an in-cluster Job). Fix: route through new
      #     /api/v1/internal/cutover/trigger endpoint which lives
      #     OUTSIDE RequireSession and validates the bearer SA token
      #     via TokenReview. The Job now mounts its projected SA
      #     token at /var/run/secrets/kubernetes.io/serviceaccount/
      #     token and sends it as `Authorization: Bearer <token>`.
      # 0.1.18: Auto-trigger readiness probe loops on 401 (#957).
      #   0.1.17 polled /api/v1/sovereign/cutover/status to check
      #   "is catalyst-api up yet?" That endpoint lives INSIDE
      #   RequireSession and returned 401 to every unauthenticated
      #   probe from the in-cluster Job. The probe treated 401 as
      #   "API not ready" → loop never broke → /internal/cutover/
      #   trigger was never called → cutover never fired (caught
      #   live on otech113 2026-05-05). Fix: poll /healthz
      #   (unauthenticated, always 200 when the process is up).
      #   Also drops the pre-flight cutoverComplete=true short-
      #   circuit since /internal/cutover/trigger is itself
      #   idempotent.
      # 0.1.19: Step-01 gitea-mirror DNS race + backoffLimit=0 (#968).
      #   0.1.18 unblocked the auto-trigger so the cutover engine fired
      #   correctly on otech115 (2026-05-05) — but Step-01 then failed
      #   within 8s with `wget: bad address gitea-http.gitea.svc.cluster.
      #   local`. The gitea Pod had reached Ready ~2-3s prior; cluster-
      #   DNS endpoint propagation was still in flight. catalyst-api
      #   stamped the Job with `backoffLimit=0` (cutover.go:584), so
      #   one DNS miss was terminal and the cutover engine aborted all
      #   8 steps. Fix is dual: (a) catalyst-api now stamps Jobs with
      #   `backoffLimit=3` so a single miss is recoverable; (b) Step-01
      #   bash script gains an explicit `nslookup` readiness loop (30 x
      #   5s) at the top, before any wget call. Both layers are needed —
      #   the in-script probe is fastest; the backoffLimit is the
      #   safety net for any other transient pre-cluster-stable race.
      # 0.1.20: Step-06 helmrepository-patches reverted by Flux (#970).
      #   0.1.19 unblocked the cutover through Step-7, but Step-08
      #   verify caught 38/38 HelmRepositories had reverted to
      #   oci://ghcr.io/openova-io despite Step-06's job logs showing
      #   `OK ${name} -> oci://harbor.<sov-fqdn>/openova-io` for each.
      #   Root cause: Step-06 only `kubectl patch`ed the live K8s
      #   objects; bootstrap-kit Kustomization reconciled YAML from
      #   local Gitea every 1m, where the YAML still declared the
      #   upstream URL, undoing each patch within ~30s. Fix: Step-06
      #   now does both phases — (a) live kubectl patches as before,
      #   then (b) clones local Gitea, sed-rewrites every
      #   clusters/_template/bootstrap-kit/*.yaml declaration of
      #   `url: oci://ghcr.io/openova-io` → local Harbor prefix,
      #   commits, and pushes. Subsequent reconciles see local Harbor
      #   as steady-state. Image bumped to alpine/k8s:1.31.4 (kubectl
      #   + git in one image; verified live on otech116).
      # 0.1.24: Step-06 phase-0 ghcr-pull harbor.<sov-fqdn> auth merge
      #   (#1184, bounded-cycle backfill). Once 0.1.20's phase-1 pivots
      #   HelmRepository URLs to oci://harbor.<sov-fqdn>/openova-io,
      #   source-controller hits a 401 on every pull because the
      #   ghcr-pull Secret only carries auth for ghcr.io and
      #   harbor.openova.io (cloud-init writes those two; harbor.<sov
      #   -fqdn> is a per-Sovereign coordinate that doesn't exist at
      #   bake time). Manually fixed on omantel 2026-05-10 (session
      #   5c468708) — bp-guacamole / bp-netbird / bp-dmz-vcluster all
      #   stuck Reconciling until `kubectl patch secret ghcr-pull` was
      #   run by hand. 0.1.24 codifies that patch as Phase-0 of Step-06
      #   so the next fresh `tofu apply` comes up GREEN with zero
      #   manual intervention. Idempotent (no-op when entry already
      #   matches), reads HARBOR_PASSWORD from the harbor-admin Secret
      #   already mirrored into `catalyst` ns by bp-harbor 1.2.14+.
      #   Adds `secrets: [update,patch]` to the runner ClusterRole.
      # 0.1.25: Step-06 Phase-0 probe brittleness fix (qa-loop bounded-
      #   cycle Wave 5 Fix #77, Gap A). 0.1.24 used kubectl jsonpath
      #   `{.data['.dockerconfigjson']}` which silently returns EMPTY
      #   because kubectl interprets the leading dot inside the bracket
      #   as a child accessor (escape `\.dockerconfigjson` would work
      #   but is a footgun). Caught live on omantel prov #7 2026-05-10:
      #   `cutover-helmrepository-patches` Job FAILED 4× with
      #   `FATAL: ghcr-pull Secret has no .dockerconfigjson key —
      #   cloud-init did not run?` despite `kubectl get secret -o yaml`
      #   showing the key present. 0.1.25 replaces the probe with
      #   `kubectl get -o json | jq -r --arg k ...` (escape-free), adds
      #   a 60s wait-loop for Reflector lag, and falls back to the
      #   source namespace (flux-system) if the local copy is still
      #   missing. Idempotent path unchanged.
      # 0.1.26: HR install/upgrade timeout 15m + values
      #   autoWaitForAPISeconds=720, autoTimeoutSeconds=840 (Fix #127).
      #   Provisions #12 + #14 wedged at phase1-watching because the
      #   HR had no explicit timeout → Helm 5m default → hit before
      #   the auto-trigger Job's 600s activeDeadline could complete.
      # 0.1.27: HR install/upgrade timeout 15m → 30m + values
      #   autoWaitForAPISeconds 720→1500s (25m wait), autoTimeoutSeconds
      #   840→1740s (29m Job deadline) (Fix #152). Prov #23 wedged
      #   identically with 3× consecutive DeadlineExceeded on the auto-
      #   trigger Job: catalyst-api had not yet become reachable inside
      #   the 14m Job deadline. Cold-start of catalyst-platform on a
      #   fresh Sovereign exceeds 14m on slow Hetzner regions; 2×
      #   headroom (29m Job, 30m HR) restores the safety margin Fix #127
      #   intended. NOTE: also bumps HR version pin from 0.1.25 → 0.1.27
      #   — Fix #127 (commit 58f518ff) bumped Chart.yaml to 0.1.26 but
      #   left this pin at 0.1.25, so the new HR-timeout/values changes
      #   never landed on any Sovereign. The pin update here is what
      #   actually delivers BOTH Fix #127 and Fix #152.
      # 0.1.28 (Fix #158, 2026-05-11): values.yaml comment cleanup —
      # platform-wide migration off bitnami/kubectl (deleted from
      # Docker Hub 2025-08). This Blueprint already uses alpine/k8s
      # + alpine since 0.1.10; no functional image change here.
      # 0.1.30 (TBD-C18, 2026-05-18): NEW step 09 (gitea-token-mint)
      # mints a real Gitea API token at cutover + patches Secret
      # sme/provisioning-github-token.GITHUB_TOKEN. The catalyst-
      # platform chart's provisioning-github-token.yaml template
      # previously mirrored the Gitea admin PASSWORD verbatim into
      # that Secret; SME provisioning then sent `Authorization: token
      # <PWD>` to Gitea which 401s ("user does not exist [uid: 0]").
      # On t22 2026-05-18: voucher checkout completed 200 + /jobs
      # redirect fired, but no Organization CR was ever created.
      # Step 09 closes the loop: DELETE-then-POST /api/v1/users/
      # gitea_admin/tokens, capture .sha1, GET /api/v1/user validate,
      # kubectl patch dest Secret, rollout-restart provisioning
      # Deployment. Order=9 (last) is fine — none of steps 02-08 read
      # the Secret and the SME provisioning service first consumes
      # the token at voucher checkout time (always postdates cutover).
      # 0.1.31 (TBD-C19, 2026-05-18): step-06 now also pivots
      # openova-catalog HelmRepository (rendered by bp-catalyst-
      # platform chart, not directly from bootstrap-kit). Adds
      # `openova-catalog` to helmRepositories.names; new Phase-1.6
      # patches the parent HelmRelease's spec.values.catalog.
      # helmRepository.url; new Phase-2.5 injects same override
      # into 13-bp-catalyst-platform.yaml in local Gitea so
      # bootstrap-kit Kustomization reconcile preserves it. Without
      # this pin bump, step-08 catches openova-catalog as the lone
      # OFFENDER ~1m after step-06 (chart re-render reverts the
      # live HR patch). Caught live on t22.omantel.biz 2026-05-18.
      # 0.1.32 (issue #1871, 2026-05-19): Step-06 helmrepository-
      # patches Job gains a NEW Phase -1 (gateway-wait) that runs
      # BEFORE Phase-0's ghcr-pull merge and Phase-1's URL rewrite.
      # The Job blocks until `gateway.networking.k8s.io/v1.Gateway
      # cilium-gateway` in `kube-system` reports `Programmed=True`,
      # which proves the Cilium Gateway has a listener serving TLS
      # on `registry.<sov-fqdn>` (the listener bp-harbor's HTTPRoute
      # attaches to). Closes the cutover↔gateway circular deadlock
      # discovered on t26 99bb823cb0513f4b (A55 diagnostic) where
      # the URL rewrite fired BEFORE the Gateway was Programmed
      # and source-controller hit TLS handshake EOF against the
      # not-yet-listening `registry.<sov-fqdn>`. Supersedes the bad
      # PR #1875 fix (which added `sovereign-tls` to dependsOn —
      # unresolvable cross-kind reference, see the dependsOn block
      # comment above). RBAC: ClusterRole gains a Rule for
      # gateway.networking.k8s.io.gateways {get,list,watch}.
      # Configurable via `.Values.gateway.{namespace,name,
      # waitTimeoutSeconds}`; default 30 min timeout safely covers
      # the slowest Hetzner cold-start observed (≈18 min).
      # 0.1.33 (TBD-A37, issue #1899, 2026-05-19): NEW post-cutover
      # continuous mirror re-sync CronJob (template 11-mirror-
      # resync-cronjob.yaml). Step-01 (gitea-mirror) only runs ONCE
      # at cutover and produces a STANDALONE local Gitea repo (PR
      # #1029); without an ongoing re-sync, upstream chart bumps
      # merged AFTER cutover never reach the Sovereign. Live
      # regression on t31 2026-05-19 (A145 verifier): sandbox-
      # controller stuck at image :8017700 from 2026-05-16 even
      # though PR #1862 had merged 2 days earlier. Chart now
      # ships a CronJob (schedule */5 default, suspend-overridable)
      # firing the same idempotent bare-clone + push --mirror
      # --force as Step 01 step (3); pre-cutover fires are no-ops.
      # No new RBAC (re-uses runner SA + reflector-mirrored gitea-
      # admin-secret). Smoke render unaffected (CronJob lacks the
      # cutover-step labels so the contract test's exactly-9-steps
      # assertion still passes).
      version: 0.1.33
      sourceRef:
        kind: HelmRepository
        name: bp-self-sovereign-cutover
        namespace: flux-system
  install:
    disableWait: true
    timeout: 30m
    remediation:
      retries: 3
  upgrade:
    disableWait: true
    timeout: 30m
    remediation:
      retries: 3
  # Per-Sovereign overrides — the chart's values.yaml carries
  # placeholders so smoke renders pass; the real coordinates land
  # here via Flux postBuild ${SOVEREIGN_FQDN} substitution.
  values:
    sovereign:
      fqdn: ${SOVEREIGN_FQDN}
      harborInternalURL: http://harbor-core.harbor.svc.cluster.local
      # NB: Harbor HTTPRoute publishes at `registry.<sov>` (see
      # `clusters/_template/bootstrap-kit/19-harbor.yaml` gateway.host),
      # NOT `harbor.<sov>`. Step-06 phase-1 rewrites every HelmRepository
      # to `oci://${harbor_host}/openova-io`, so this MUST be the public
      # hostname that actually answers — `registry.${SOVEREIGN_FQDN}`.
      # Pre-2026-05-18 this said `harbor.${SOVEREIGN_FQDN}`, which no
      # HTTPRoute matched → all post-pivot OCI pulls EOF → bp-sandbox HR
      # never Ready → bootstrap-kit Ks stuck (chicken-and-egg). See
      # t20 debug matrix.
      harborPublicURL: https://registry.${SOVEREIGN_FQDN}
      giteaInternalURL: http://gitea-http.gitea.svc.cluster.local:3000
      giteaPublicURL: https://gitea.${SOVEREIGN_FQDN}
