# bp-cluster-autoscaler-hcloud — Catalyst bootstrap-kit Blueprint #50
# (Tier 5 — Scaling/Resilience). Slot 40 was already forward-declared
# for bp-llm-gateway in scripts/expected-bootstrap-deps.yaml; this
# blueprint lands at slot 50 — after the W2.K4 cohort + slot 49
# (bp-cert-manager-powerdns-webhook) — to preserve the existing
# numbering plan.
#
# Adds and removes Hetzner Cloud worker nodes on demand in response to
# `FailedScheduling` events on the Sovereign's k3s cluster. Bounded by
# the `min`/`max` node-group config the operator picked at launch.
#
# Live evidence motivating this blueprint (issue #767):
#   otech92 — 2× cpx32 workers couldn't fit external-secrets-webhook
#   because the bootstrap-kit's RAM aggregate (~14 GB across 35
#   HelmReleases) exceeded the 2× 8 GB pool the operator chose. With
#   cluster-autoscaler the Sovereign would have grown the pool to a
#   third worker automatically.
#
# Wrapper chart: platform/cluster-autoscaler-hcloud/chart/ — umbrella
# over upstream kubernetes/autoscaler cluster-autoscaler chart 9.46.6
# (appVersion 1.32.0). Catalyst-curated values flow under the
# `cluster-autoscaler:` key + a vendor-agnostic
# `clusterAutoscalerHcloud.*` block that ships the namespace-local
# Hetzner-API-token Secret (`hcloud-token`).
#
# Reconciled by: Flux on the new Sovereign's k3s control plane.
#
# Hetzner-token wiring (mirrors the velero/harbor object-storage pattern
# in 19-harbor.yaml + 34-velero.yaml):
#   - cloud-init writes `flux-system/cloud-credentials` Secret with the
#     `hcloud-token` key (see infra/hetzner/cloudinit-control-plane.tftpl
#     §"cloud-credentials-secret"). That Secret is the canonical Hetzner-
#     API-token holder for every Day-2 mutation seam (Crossplane provider-
#     hcloud, this autoscaler, future hcloud Floating-IP claims).
#   - This HelmRelease lifts the `hcloud-token` value into the umbrella
#     chart's `clusterAutoscalerHcloud.hcloudToken` value via Flux
#     `valuesFrom`. The umbrella chart then synthesises a namespace-local
#     `cluster-autoscaler/hcloud-token` Secret (templates/hetzner-token-
#     secret.yaml) the upstream chart's `extraEnvSecrets.HCLOUD_TOKEN`
#     wiring binds as the deployment's HCLOUD_TOKEN env var.
#
# dependsOn: (none) — cluster-autoscaler is independent of every other
# bootstrap-kit blueprint at install time. The cloud-credentials Secret
# is provisioned by cloud-init BEFORE Flux installs anything.

---
apiVersion: v1
kind: Namespace
metadata:
  name: cluster-autoscaler
  labels:
    catalyst.openova.io/sovereign: ${SOVEREIGN_FQDN}
---
apiVersion: source.toolkit.fluxcd.io/v1beta2
kind: HelmRepository
metadata:
  name: bp-cluster-autoscaler-hcloud
  namespace: flux-system
spec:
  type: oci
  interval: 15m
  url: oci://ghcr.io/openova-io
  secretRef:
    name: ghcr-pull
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: bp-cluster-autoscaler-hcloud
  namespace: flux-system
  labels:
    catalyst.openova.io/slot: "50"
spec:
  interval: 15m
  releaseName: cluster-autoscaler
  targetNamespace: cluster-autoscaler
  chart:
    spec:
      chart: bp-cluster-autoscaler-hcloud
      # 1.3.0 — qa-loop chroot-canvas Fix: wire HCLOUD_NETWORK /
      # HCLOUD_FIREWALL / HCLOUD_SSH_KEY env vars onto the autoscaler
      # deployment so scale-up VMs land on the Phase-0 private network +
      # firewall + ssh-key, identical to Phase-0 workers. Without these
      # the autoscaler-spawned VMs only receive a public IP, the worker
      # cloud-init's `K3S_URL=https://10.0.1.2:6443` is unreachable, the
      # k3s agent join silently fails, and every scale-up times out at
      # 15m → backoff. Live evidence: prov #38/#39/#41/#43 omantel.biz.
      #
      # 1.2.0 — qa-loop Wave 5 Fix #79 Gap D: chart-derived
      # HCLOUD_CLUSTER_CONFIG fallback. When the per-Sovereign
      # cloud-init has not stamped the `hcloud-cloud-init` key into
      # `flux-system/cloud-credentials`, the chart synthesises a
      # minimal HCLOUD_CLUSTER_CONFIG JSON from the existing
      # `cluster-autoscaler.autoscalingGroups[]` so the autoscaler
      # never FATALs with the generic
      # "HCLOUD_CLUSTER_CONFIG or HCLOUD_CLOUD_INIT is not specified".
      version: 1.3.0
      sourceRef:
        kind: HelmRepository
        name: bp-cluster-autoscaler-hcloud
        namespace: flux-system
  # Event-driven install: cluster-autoscaler is a single Deployment +
  # ServiceAccount + RBAC. Helm install completes when manifests apply;
  # the binary's Hetzner-API connectivity check is a runtime concern,
  # not a Helm-wait concern. disableWait keeps Flux's Ready signal
  # aligned with manifest apply.
  install:
    timeout: 15m
    disableWait: true
    remediation:
      retries: 3
  upgrade:
    timeout: 15m
    disableWait: true
    remediation:
      retries: 3
  # ── Hetzner-token + node-bootstrap wiring (issue #921) ─────────────
  # Pulls keys from the canonical `flux-system/cloud-credentials`
  # Secret cloud-init writes at Phase 0
  # (infra/hetzner/cloudinit-control-plane.tftpl §"cloud-credentials-
  # secret"):
  #   - hcloud-token       → API token (mandatory)
  #   - hcloud-cloud-init  → base64(cloud-init.yaml) — the autoscaler-
  #                          spawned worker's bootstrap, identical to the
  #                          Phase-0 worker user_data. Required by
  #                          cluster-autoscaler 1.32.x's Hetzner provider
  #                          (HCLOUD_CLOUD_INIT env var) — without it the
  #                          autoscaler Pod exits at startup with FATAL
  #                          "HCLOUD_CLUSTER_CONFIG or HCLOUD_CLOUD_INIT
  #                          is not specified".
  # Flux dereferences `valuesFrom` at HelmRelease apply time, so the
  # plaintext payloads never appear in this committed manifest.
  #
  # The chart's templates/hetzner-node-config-secret.yaml renders these
  # values into a namespace-local `cluster-autoscaler/hetzner-node-config`
  # Secret which the upstream chart's `extraEnvSecrets.HCLOUD_CLOUD_INIT`
  # binding lifts onto the deployment's env.
  valuesFrom:
    - kind: Secret
      name: cloud-credentials
      valuesKey: hcloud-token
      targetPath: clusterAutoscalerHcloud.hcloudToken
    - kind: Secret
      name: cloud-credentials
      valuesKey: hcloud-cloud-init
      targetPath: clusterAutoscalerHcloud.cloudInit
      # When older Sovereigns provisioned BEFORE issue #921 lack the
      # hcloud-cloud-init key, Flux skips this entry rather than failing
      # the entire HelmRelease — the chart's empty-string default keeps
      # the upstream Deployment shape valid (the autoscaler will still
      # FATAL at startup, surfacing the missing-cloud-init in Pod logs;
      # operators rotate by re-running cloud-init or by patching
      # cloud-credentials directly).
      optional: true
    # ── Issue #1778 — Hetzner network/firewall/ssh-key attachment ──────
    # The cluster-autoscaler-hcloud provider only attaches scale-up
    # servers to an existing private network when HCLOUD_NETWORK is set
    # at startup. Without it, the Phase-0 workers (which join via
    # 10.0.1.2:6443 on the private subnet) and the autoscaler-spawned
    # workers (which only have a public IP) live on different network
    # planes — the autoscaler VMs cannot reach the apiserver private
    # endpoint, the k3s agent join times out, the node never registers,
    # the autoscaler hits a 15m scale-up timeout and enters backoff.
    # Names are written by cloud-init (see
    # infra/hetzner/cloudinit-control-plane.tftpl `hcloud-network-name`
    # etc.) so the autoscaler attaches every scale-up VM to the
    # SAME network + firewall + ssh-key the Phase-0 Tofu module created.
    # The chart's values.yaml default of empty-string keeps the upstream
    # deployment shape valid for legacy Sovereigns whose cloud-init
    # never stamped these keys; on those Sovereigns Flux just skips the
    # entry (optional: true) and the autoscaler runs in its pre-#1778
    # shape (still broken, but no Helm render error).
    - kind: Secret
      name: cloud-credentials
      valuesKey: hcloud-network-name
      targetPath: cluster-autoscaler.extraEnv.HCLOUD_NETWORK
      optional: true
    - kind: Secret
      name: cloud-credentials
      valuesKey: hcloud-firewall-name
      targetPath: cluster-autoscaler.extraEnv.HCLOUD_FIREWALL
      optional: true
    - kind: Secret
      name: cloud-credentials
      valuesKey: hcloud-ssh-key-name
      targetPath: cluster-autoscaler.extraEnv.HCLOUD_SSH_KEY
      optional: true
  # Per-Sovereign baseline values. clusters/<sovereign>/bootstrap-kit/
  # 40-cluster-autoscaler.yaml MAY override `autoscalingGroups` to set
  # the actual instanceType + region + min/max + name the Tofu module
  # provisioned at Phase 0. The defaults below match the canonical
  # otechN topology (cpx32 in fsn1, min 2 / max 10) so a vanilla
  # Sovereign that forgets to patch this still gets a sensible
  # autoscaler.
  values:
    cluster-autoscaler:
      autoscalingGroups:
        - name: workers
          instanceType: cpx32
          region: fsn1
          minSize: 2
          maxSize: 10
