# bp-cilium — Catalyst bootstrap-kit Blueprint. CNI must come first; k3s started with --flannel-backend=none precisely so Cilium can take over.
#
# Wrapper chart: platform/cilium/chart/
# Catalyst-curated values: platform/cilium/chart/values.yaml
# Reconciled by: Flux on the new Sovereign's k3s control plane.

---
# kube-system is built into every Kubernetes cluster — never re-declare it.
# Earlier revisions of 01-cilium.yaml AND 05-sealed-secrets.yaml both
# declared it, which collided when kustomize tried to merge the two:
#   "may not add resource with an already registered id:
#    Namespace.v1.[noGrp]/kube-system.[noNs]"
# This Blueprint installs Cilium INTO kube-system; the HelmRelease's
# targetNamespace field below is sufficient.
apiVersion: source.toolkit.fluxcd.io/v1beta2
kind: HelmRepository
metadata:
  name: bp-cilium
  namespace: flux-system
spec:
  type: oci
  interval: 15m
  url: oci://ghcr.io/openova-io
  secretRef:
    name: ghcr-pull
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: bp-cilium
  namespace: flux-system
  labels:
    catalyst.openova.io/slot: "01"
spec:
  interval: 15m
  releaseName: cilium
  targetNamespace: kube-system
  chart:
    spec:
      chart: bp-cilium
      # 1.3.3 (qa-loop iter-16 Fix #70): Hubble UI HTTPRoute defaults
      # corrected — gatewayRef.namespace=kube-system (was the stale
      # cilium-gateway), serviceRef.namespace=kube-system (was the stale
      # cilium), plus chart auto-derives hubble.<sovereignFQDN> when only
      # SOVEREIGN_FQDN is provided. Combined with the bootstrap-kit
      # default flip below (HUBBLE_ENABLED=true, hubble.relay/ui enabled,
      # SOVEREIGN_FQDN forwarded), every Sovereign exposes Hubble UI at
      # https://hubble.<sovereignFQDN>/ out of the box. TC-289 NXDOMAIN
      # is closed because external-dns now sees the HTTPRoute hostname
      # and writes the A record into PowerDNS.
      # 1.3.1 (qa-loop iter-12 Fix #54 Workstream 2): bpf.preallocateMaps=true
      # + socketLB.hostNamespaceOnly=true defaults so fresh worker pods can
      # resolve DNS reliably on first-join (cilium/cilium#28456 mitigation).
      # 1.3.0 (qa-loop iter-12 Fix #53C+D): adds the Hubble UI HTTPRoute
      # overlay (slice H7 #1095) that the catalystOverlay.hubbleUI block
      # below depends on.
      version: 1.3.3
      sourceRef:
        kind: HelmRepository
        name: bp-cilium
        namespace: flux-system
  # Event-driven install: Helm completes when manifests apply, not when
  # cilium-agent reaches Ready (agent waits for envoyconfig CRDs that the
  # SAME chart installs — legitimate slow-Ready). Replaces blanket
  # spec.timeout: 15m band-aid from PR #221.
  install:
    timeout: 15m
    disableWait: true
    remediation:
      retries: 3
  upgrade:
    timeout: 15m
    disableWait: true
    remediation:
      retries: 3
  values:
    cilium:
      # Phase-8a bug #15 (otech8 deployment 1bfc46347564467b 2026-05-01):
      # cilium-agent waits forever for the operator to register
      # ciliumenvoyconfigs + ciliumclusterwideenvoyconfigs CRDs.
      # Setting `envoy.enabled: true` (chart-level) runs Envoy as a separate
      # daemonset but does NOT register those CRDs — that requires
      # `envoyConfig.enabled: true`, a separate upstream chart toggle.
      # Without it, the agent's node taint `node.cilium.io/agent-not-ready`
      # never lifts and every other HelmRelease (37 of them) blocks on its
      # dependsOn chain.
      envoyConfig:
        enabled: true
      l7Proxy: true
      prometheus:
        enabled: false
        serviceMonitor:
          enabled: false
      hubble:
        metrics:
          # `null` (NOT [] and NOT a populated list) suppresses the
          # upstream chart's metrics ServiceMonitor render. Hubble flow
          # collection still works for Hubble Relay/UI without a
          # ServiceMonitor — that pulls the kube-prometheus-stack CRDs
          # which do not exist on a fresh Sovereign at bp-cilium install
          # time. Operators flip metrics on once
          # bp-kube-prometheus-stack is reconciled (issue #182).
          enabled: null
          serviceMonitor:
            enabled: false
        # qa-loop iter-16 Fix #70: default Hubble UI ON for every
        # Sovereign so TC-289 (https://hubble.<fqdn>/) resolves out of
        # the box. Hubble flow telemetry is the canonical L3-L7 visibility
        # surface for the Catalyst control plane (EPIC-5 #1100); shipping
        # it default-OFF made every Sovereign blind by default and
        # required a per-Sovereign overlay touch that nobody remembered
        # to wire. Per-Sovereign overlay can still set HUBBLE_ENABLED=false
        # for fully air-gapped lab Sovereigns where Relay traffic to
        # cilium-agents is not desired.
        relay:
          enabled: ${HUBBLE_ENABLED:=true}
        ui:
          enabled: ${HUBBLE_ENABLED:=true}

      # qa-loop iter-12 Fix #53C: BGP control plane (default off, opt-in
      # via BGP_ENABLED=true). Per ADR-0001 §9 the BGP control plane is the
      # canonical path for Sovereign-to-customer-router prefix advertisement
      # (LoadBalancer VIPs, Pod CIDRs to customer's existing core network).
      bgpControlPlane:
        enabled: ${BGP_ENABLED:=false}

      # ── Cilium ClusterMesh — multi-region peering ──────────────────
      #
      # Per ADR-0001 §9 + EPIC-6 #1101 (multi-region active-hotstandby DR),
      # ClusterMesh is the canonical inter-region transport for replication
      # and Service-of-type-global traffic between Sovereign peer clusters.
      #
      # cluster.name + cluster.id are PER-SOVEREIGN anchors; per
      # docs/INVIOLABLE-PRINCIPLES.md #4 (never hardcode), they MUST come
      # from the Flux Kustomization's postBuild.substitute block — which
      # in turn flows from infra/hetzner/cloudinit-control-plane.tftpl
      # (CLUSTER_MESH_NAME, CLUSTER_MESH_ID) and ultimately from the
      # operator-supplied request.cluster_mesh_name + cluster_mesh_id at
      # provision time. Mesh registry: docs/CLUSTERMESH-CLUSTER-IDS.md
      # tracks the cluster.id allocation across the OpenOva fleet.
      #
      # NodePort 32379: clustermesh-apiserver Pod is exposed on every
      # Cilium node so peers reach it over the Hetzner private network on
      # `<cp-private-ip>:32379` WITHOUT requiring a Hetzner LoadBalancer
      # per peer (LB count is project-quota'd). Hetzner firewall must
      # open 32379/tcp from peer Sovereigns' Hetzner CIDRs.
      #
      # A Sovereign that does NOT join a mesh leaves CLUSTER_MESH_NAME
      # empty (Flux envsubst rule: ${VAR:=default} -> "default" when
      # unset/empty). The cilium subchart accepts an empty cluster.name
      # provided cluster.id stays 0; the clustermesh-apiserver Pod still
      # runs but no peer connects (single-cluster no-op).
      cluster:
        name: ${CLUSTER_MESH_NAME:=}
        id: ${CLUSTER_MESH_ID:=0}
      clustermesh:
        useAPIServer: true
        apiserver:
          service:
            # qa-loop iter-12 Fix #53D: ${CLUSTERMESH_SERVICE_TYPE:=NodePort}
            # Default NodePort keeps backward-compat (existing Sovereigns
            # don't break). Sovereigns that need cross-region peering set
            # CLUSTERMESH_SERVICE_TYPE=LoadBalancer in their bootstrap-kit
            # Kustomization substitute (requires Hetzner CCM bootstrap).
            type: ${CLUSTERMESH_SERVICE_TYPE:=NodePort}
            nodePort: 32379

    # ── Catalyst overlay templates (chart/templates/) ────────────────────
    # qa-loop iter-16 Fix #70: Hubble UI HTTPRoute now defaults ON for
    # every Sovereign. The chart auto-derives hostname `hubble.${SOVEREIGN_FQDN}`
    # so the operator only needs the SOVEREIGN_FQDN substitute (already
    # mandatory for every Sovereign — see clusters/_template/bootstrap-kit/
    # 13-bp-catalyst-platform.yaml `host: console.${SOVEREIGN_FQDN}`).
    # Per-Sovereign overlay can still:
    #   - HUBBLE_ENABLED=false  → disable Hubble UI on this Sovereign
    #   - HUBBLE_HOSTNAME=...   → override the auto-derived hostname
    #   - HUBBLE_AUTH=oidc      → enable OIDC enforcement once the
    #                              Keycloak realm wires the hubble-ui client
    catalystOverlay:
      hubbleUI:
        enabled: ${HUBBLE_ENABLED:=true}
        # Explicit override; empty triggers the chart to derive
        # `hubble.${SOVEREIGN_FQDN}` from sovereignFQDN below.
        hostname: ${HUBBLE_HOSTNAME:=}
        sovereignFQDN: ${SOVEREIGN_FQDN}
        gatewayRef:
          # The Sovereign Gateway lives in kube-system — installed by
          # clusters/_template/sovereign-tls/cilium-gateway.yaml. Every
          # other bootstrap-kit HTTPRoute (gitea, auth, grafana, harbor,
          # openbao, powerdns, console/catalyst-platform) attaches to
          # cilium-gateway/kube-system; this overlay matches.
          name: cilium-gateway
          namespace: kube-system
        # `none` until the Keycloak `hubble-ui` OIDC client is wired by
        # bp-keycloak realm-config; flip to `oidc` per per-Sovereign
        # overlay once that lands. Until then Hubble UI is publicly
        # reachable — acceptable for the in-progress qa-loop iter-16
        # observability slice; lock down before production handover via
        # HUBBLE_AUTH=oidc.
        auth: ${HUBBLE_AUTH:=none}
        serviceRef:
          name: hubble-ui
          namespace: kube-system
          port: 80
---
# ─── Per-Sovereign Gateway API resources (issue #387) ────────────────────
#
# Cilium owns the GatewayClass (`cilium`) installed by the chart above
# (gatewayAPI.enabled=true, envoy.enabled=true in platform/cilium/chart/
# values.yaml). The single per-Sovereign Gateway listening on
# *.${SOVEREIGN_FQDN}:443 lives here so it boots alongside the CNI
# without needing a new bootstrap-kit slot — every Sovereign HTTP
# blueprint (catalyst-platform, gitea, keycloak, harbor, grafana,
# openbao, powerdns) attaches its HTTPRoute to this Gateway via
# parentRefs.
#
# TLS material: a wildcard Certificate is requested from
# letsencrypt-dns01-prod-powerdns (cert-manager + bp-cert-manager-
# powerdns-webhook from #373; webhook calls contabo's central PowerDNS
# at https://pdns.openova.io). The resulting Secret
# `sovereign-wildcard-tls` is referenced by the Gateway listener.
#
# Cross-namespace HTTPRoute attachment: allowedRoutes.namespaces.from=All
# permits every blueprint namespace (catalyst-system, gitea, keycloak,
# harbor, grafana-system, openbao, powerdns-system) to bind without a
# ReferenceGrant. This matches the Catalyst single-tenant Sovereign
# model — cross-tenant isolation is enforced by per-tenant vClusters
# (bp-vcluster), not by Gateway-level RBAC.
#
# Per ADR-0001 §9.4 and docs/INVIOLABLE-PRINCIPLES.md #4: this resource
# only renders when ${SOVEREIGN_FQDN} is set by Flux envsubst at the
# Sovereign apply time — contabo's bootstrap path does NOT include this
# template, so Traefik continues to serve console.openova.io/nova
# unchanged.
