values.yaml

# Default values for openstudio-server.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.


# Global image registry settings - used by imageWithRegistry helper
# Set these per-provider (e.g., openstack/values-openstack.yaml) for private registries
global:
  images:
    registry: ""
    repositoryPrefix: ""

# Local image registry (optional, deployed as subchart like nfs-server-provisioner)
# When enabled, a local Docker registry (distribution/registry:2) is deployed in-cluster.
# If rewriteImages is true, all workload images are rewritten to pull from the local registry.
localRegistry:
  enabled: false
  # Subchart values (passed through to charts/local-registry)
  image: "registry:2"
  resources: {}
  persistence:
    enabled: true
    storageClass: "ssd"
    size: 10Gi
  # Image rewrite settings
  rewriteImages: true
  # Hostname override (for external registry use instead of deployed subchart)
  hostname: ""
  port: 5000

# containerd node-level registry configuration (certs.d/hosts.toml).
#
# Problem this solves: containerd on each node reads its own
# /etc/containerd/certs.d/<host>/hosts.toml to decide how to reach a given
# registry host (e.g. an insecure/plain-HTTP local registry, or a mirror
# fronting a registry that's otherwise unreachable/rate-limited). That file
# lives on the node's local disk, NOT in-cluster state -- so it does not
# exist on any node the cluster autoscaler creates after the fact, and is
# lost if a node is replaced. Manually SSHing/kubectl-debugging it onto nodes
# is not durable across scale-up events or cluster rebuilds.
#
# Fix: a DaemonSet (templates/containerd-registry-config/*) runs on every
# web/worker node, writes the hosts.toml file(s) below to the node's
# /etc/containerd/certs.d/, and periodically re-writes them so the DaemonSet
# both self-heals onto newly-autoscaled nodes (normal DaemonSet scheduling)
# and repairs any file that gets reset/lost on an existing node.
#
# This only ever needs the SERVICE's ClusterIP notion for the local
# registry, which is resolved from the same localRegistry.hostname/port
# values already used by the imageWithRegistry helper -- so there is a
# single source of truth; you don't set the IP twice.
containerdRegistryConfig:
  # Master toggle. Auto-enabled (effectively true) whenever the local
  # registry rewrite is active, even if left false here -- see
  # openstudio.containerdRegistryConfigEnabled helper. Set explicitly to
  # true/false to force it on/off regardless of localRegistry settings.
  enabled: false
  image: "docker.io/calico/node:v3.29.3" # any small always-on-every-node image with a shell; already pre-pulled on most clusters as part of the CNI, so this DaemonSet doesn't add a new pull dependency
  # How often (seconds) to re-write the hosts.toml files, so a reset/lost
  # file (e.g. after node reboot without persistent /etc, or manual edit)
  # self-heals without needing a pod restart.
  refreshIntervalSeconds: 300
  resources:
    requests:
      cpu: 50m
      memory: 64Mi
    limits:
      cpu: 200m
      memory: 512Mi
  # Additional registry mirrors to configure, beyond the local registry
  # (which is handled automatically from localRegistry.* above when
  # rewriteImages is enabled). Each entry writes
  # /etc/containerd/certs.d/<host>/hosts.toml on every web/worker node.
  #
  # Example (mirroring a registry that's rate-limiting or unreachable):
  #   extraMirrors:
  #     - host: "pulp-dev.hpc.nlr.gov"
  #       server: "https://pulp-dev.hpc.nlr.gov"
  #       mirror: "http://192.168.66.44:30500"
  #       capabilities: ["pull", "resolve"]
  extraMirrors: []
  # Pre-pull (warm) workload images into containerd's content store directly
  # via `ctr`, bypassing kubelet entirely.
  #
  # Problem this solves: kubelet's default registryPullQPS=5/registryBurst=10
  # with serializeImagePulls=true throttles and SERIALIZES every image pull
  # per node, regardless of how fast the registry itself can actually serve
  # images. Under a large/fast HPA scale-up (many new pods landing on a
  # freshly-joined node at once), this produces a wall of concurrent
  # "Failed to pull image ... pull QPS exceeded" / ImagePullBackOff events
  # even though the local registry has plenty of headroom -- kubelet's own
  # image manager is the bottleneck, not the network or the registry.
  # Talking to containerd directly via its socket (as this DaemonSet does)
  # goes around kubelet's image manager entirely, so a single `ctr images
  # pull` per image per node completes in ~0.1-0.3s once the layers are in
  # containerd's content store (confirmed on this cluster), and every
  # subsequent kubelet-initiated pull for that image on that node is
  # instant (content-addressed layers already present) instead of
  # contending for the QPS/burst budget.
  #
  # Requires hostPID plus bind-mounting the node's containerd.sock and `ctr`
  # binary (both standard on every containerd-based node; no extra
  # dependency installed on the node). Runs once at DaemonSet pod startup
  # (i.e. once per node join) rather than on every refresh loop, since
  # images rarely change and repeated pulls of the same content-addressed
  # layers are cheap no-ops.
  prewarmImages:
    enabled: false
    # containerd's socket + ctr binary paths on the host. Standard for any
    # containerd-based node (matches every node observed on this cluster);
    # override if your OS/distro places these elsewhere.
    containerdSockPath: "/run/containerd/containerd.sock"
    ctrBinaryPath: "/usr/local/bin/ctr"
    # Explicit extra images to pre-pull, beyond the ones automatically
    # derived from web/web_background/worker/rserve/db/redis container
    # images (see openstudio.prewarmImageList helper). Use this for images
    # not otherwise covered, e.g. an extra sidecar.
    extraImages: []

# Hook configuration
# Pre-delete hook ("nfs-client-cleanup" Job) drains all release workloads except
# the NFS server provisioner subchart, waits for client pods to be gone, and
# force-deletes leftovers. It keeps the NFS server up until every client pod has
# unmounted, so pods cannot hang in Terminating on a dead NFS mount.
hooks:
  preDeleteCleanup:
    image: "bitnami/kubectl:latest" # single source of truth for the hook image
    # Total Job budget: image pull + workload deletion + podWaitTimeoutSeconds +
    # force-delete + settle. Keep comfortably above podWaitTimeoutSeconds.
    activeDeadlineSeconds: 300
    # How long to wait for client pods to drain before force-deleting them.
    podWaitTimeoutSeconds: 120

# Set to google, aws, azure, or openstack
provider:
  name: ""

# Node label used by pod scheduling affinity (see templates/_helpers.tpl,
# "openstudio.nodeGroupAffinity"). Defaults match this chart's original
# hardcoded behavior exactly, so existing installs are unaffected. Override
# per-provider (see openstack/values-openstack.yaml) if your cluster labels
# node groups differently, e.g. Cluster API/Azimuth clusters auto-label nodes
# "capi.stackhpc.com/node-group: web|worker" instead of "nodegroup: web-group".
node_group:
  label_key: "nodegroup"
  web_value: "web-group"
  worker_value: "worker-group"
  # "required" or "preferred". Use "preferred" if newly-autoscaled nodes may
  # take a moment to receive their node-group label after joining the
  # cluster, to avoid pods staying stuck Pending during that window.
  affinity_mode: "required"

cluster:
  name: "openstudio-server"

nfs-server-provisioner:
  persistence:
    enabled: true
    storageClass: "ssd"
    size: 110Gi
  storageClass:
    allowVolumeExpansion: false
    # NFSv4 is stateful and does not tolerate being fronted by a Service
    # ClusterIP (client callback/lease renewal fails, mounts hang/fail).
    # vers=3 has fixed mountd/nlockmgr/statd/rpcbind ports and is what this
    # provisioner is actually tested against.
    mountOptions:
      - vers=3
      - sync
db:
  name:  "db"
  label: "db"
  username: "openstudio"
  password: "openstudio"
  container:
    name: "mongo-db"
    image: "mongo:6.0.7"
    resources:
      requests:
        cpu: 1
        memory: "4Gi"
    ports:
      db_port: 27017
  persistence:
    enabled: true
    storageClass: "ssd"
    size: 100Gi
    accessModes:
      - "ReadWriteOnce"

load_balancer:
  name: "ingress-load-balancer"
  externalTrafficPolicy: "Local"
  label: "web"
  internal: false
  ports:
    http_name: "http"
    http_port: 80
    http_protocol: "TCP"
    https_name: "https"
    https_port: 443
    https_protocol: "TCP"

nfs:
  name: "nfs"

nfs_pvc:
  name: "nfs-pvc"
  accessModes:
    - "ReadWriteMany"
  storage_class: "nfs"
  storage: "100Gi"


redis:
  name: "redis"
  label: "redis"
  password: "openstudio"
  # Max concurrent client connections for the redis server (--maxclients).
  # Tune upward for deployments with many worker pods.
  maxclients: 10000
  container:
    name: "redis"
    image: "redis:6.0.9"
    resources:
      requests:
        cpu: 0.25
        memory: "1Gi"
    port: 6379
  persistence:
    enabled: true
    storageClass: "ssd"
    size: 100Gi
    accessModes:
      - "ReadWriteOnce"

redis_svc:
  name: "queue"
  label: "redis"
  port: 6379
  url: "redis://:openstudio@queue:6379"

rserve:
  name: "rserve"
  label: "rserve"
  container:
    name: "rserve"
    image: "nrel/openstudio-rserve:3.8.0-1"
    resources:
      requests:
        cpu: 1
        memory: "2Gi"

rserve_svc:
  name: "rserve"
  label: "rserve"
  port: 6311

web_background:
  name: "web-background"
  label: "web-background"
  replicas: 1
  number_of_workers: "30"
  container:
    name: "web-background"
    image: "nrel/openstudio-server:3.8.0-1"
    resources:
      requests:
        cpu: 0.25
        memory: "512Mi"

web:
  name: "web"
  label: "web"
  secret_key_value: "c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473"
  # passenger_max_request_queue_size: "1600"
  # Use this formula (1024 * memory of web pod in Gi * 0.75) / memory per passenger process
  # passenger_max_pool_size: "21"
  # This value is typically between 150 and 400, find this by ssh into web pod and doing `passenger-status`
  passenger_memory_per_process: 250
  container:
    name: "web"
    image: "nrel/openstudio-server:3.8.0-1"
    resources:
      requests:
        cpu: 1
        memory: "2Gi"
    port:
     http: 80
     https: 443

# For this to work and have more than 1 web pod we'll need to implement
# a distributed file locking scheme such a https://github.com/antirez/redlock-rbs
# as even with using NFS via sync it cannot guarantee file locking using multiple clients
web_hpa:
  name: "web"
  minReplicas: 1
  maxReplicas: 1
  targetCPUUtilizationPercentage: 50

web_svc:
  name: "web"
  label: "web"
  ports:
    http: 80
    https: 443

worker:
  name: "worker"
  label: "worker"
  container:
    name: "worker"
    image: "nrel/openstudio-server:3.8.0-1"
    resources:
      requests:
        cpu: 700m
        memory: "900Mi"
    terminationGracePeriodSeconds: 5200
    # Cap for worker scratch space under /mnt/openstudio (emptyDir, backed
    # by the node's local disk). Without a cap, one worker writing more
    # than expected can fill the node's disk and trigger DiskPressure
    # eviction of every pod on that node, not just the offending one.
    # Leave empty ("") for no limit (original behavior). Example: "8Gi".
    emptyDirSizeLimit: ""

worker_hpa:
  name: "worker"
  minReplicas: 2
  maxReplicas: 20
  targetCPUUtilizationPercentage: 50
  stabilizationWindowSeconds: 3600
  # Stabilization window for scale-UP (seconds). Prevents a single transient
  # CPU sample above target (pod startup, requeue storm after a scale-down)
  # from driving the HPA to maxReplicas via the scale-up policy before
  # sustained load is confirmed. Defaults to stabilizationWindowSeconds
  # when unset (backwards compatible).
  scaleUpStabilizationWindowSeconds: 300
  # Scale-up burst size, in pods per periodSeconds=60.
  scalePolicyValue: 25
  # Optional separate cap for scaleDown, in pods per periodSeconds=60.
  # Defaults to scalePolicyValue (original behavior: scale-up and scale-down
  # share the same burst size) when unset. Scale-down bursts of hundreds of
  # simultaneous pod terminations risk overwhelming a node's container
  # runtime kill pathway if many land on the same node at once; set this
  # lower than scalePolicyValue to throttle scale-down specifically without
  # slowing scale-up responsiveness.
  scaleDownPolicyValue: null

# Cluster Autoscaler configuration
clusterAutoscaler:
  image: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.6"
  # Host path for the CA bundle, mounted read-only into the container at
  # /etc/ssl/certs/ca-certificates.crt so the autoscaler can make TLS calls
  # to the cloud provider API.
  #
  # Default matches Debian/Ubuntu (ca-certificates package layout). RHEL/
  # CentOS/Fedora-based nodes use a different path/filename
  # ("/etc/ssl/certs/ca-bundle.crt" via the ca-certificates rpm) -- override
  # this value for those distros. Getting this wrong causes the pod to
  # CrashLoopBackOff with:
  #   "OCI runtime create failed: ... mount ...: not a directory: unknown"
  # (an empty directory gets created+mounted at the source hostPath instead
  # of the expected file when the configured path doesn't exist).
  caCertsHostPath: "/etc/ssl/certs/ca-certificates.crt"