Server Configuration

{
  "cluster": {
    "id": "example-cluster", // Cluster ID (lowercase alphanumeric + hyphens)
    // Optional: separate storage for cluster-scoped data. If omitted, the shard bucket is used with the "cluster/" prefix.
    "storage": {
      "provider": "s3",
      "bucket": "example-nstance-state",
      "region": "us-west-2",
      "prefix": "cluster/"
    },
    "secrets": {
      // AWS default. Names are appended below this Parameter Store path.
      "provider": "aws-parameter-store",
      "prefix": "/example-cluster/",
      // Optional: secret cache TTL. Omit or set to 0 to disable caching.
      "cache_ttl": "5m"
    },
    // Optional: cluster-wide leader election timings. Omit to use server defaults.
    "leader_election": {
      "enabled": true,
      "frequent_interval": "5s",
      "infrequent_interval": "30s",
      "leader_timeout": "15s"
    }
  },
  "shard": {
    "id": "us-west-2a", // Shard ID (lowercase alphanumeric + hyphens)
    "infra": {
      "provider": "aws",
      "region": "us-west-2",
      "zone": "us-west-2a", // aka CAPI "Failure Domain"
      "options": {
        "profile": "production"
      }
    },
    "bind": {
      "health_addr": "0.0.0.0:8990",       // HTTP health endpoint for ASG/LB health checks (503 until config is loaded, then 200)
      "election_addr": "0.0.0.0:8991",     // HTTPS leader election health endpoint for peer-to-peer leader election health checks
      "registration_addr": "0.0.0.0:8992", // Agent/Operator registration (anonymous)
      "operator_addr": "0.0.0.0:8993",     // Operator connections (authenticated)
      "agent_addr": "0.0.0.0:8994"         // Agent connections (authenticated)
    },
    "advertise": {
      "health_addr": "10.0.0.1:8990",       // HTTP health endpoint advertised for ASG/LB health checks and monitoring
      "election_addr": "10.0.0.1:8991",     // HTTPS leader election health endpoint advertised to peer servers
      "registration_addr": "10.0.0.1:8992", // Address used by Nstance Agents and Nstance Operator to register
      "operator_addr": "10.0.0.1:8993",     // Address used by Nstance Operator to connect
      "agent_addr": "10.0.0.1:8994"         // Address used by Nstance Agents to connect
      // If the host part is empty, "0.0.0.0", or "::", it is auto-detected from the primary network interface.
    },
    // Optional: stable leader network configuration for shard leadership.
    "leader_network": {
      // AWS requires both ip and interface_id (ENI ID). Google Cloud uses ip and leaves interface_id empty.
      "ip": "10.0.0.100",                    // Stable leader IP address (ENI private IP for AWS, reserved IP for Google Cloud)
      "interface_id": "eni-0abc123def456789" // AWS ENI ID (required for AWS, not used for Google Cloud)
    },
    "subnet_pools": {
      "nat": ["subnet-00000000"],                      // Public service subnet for NAT VMs
      "control-plane": ["subnet-12345678"],            // Maps subnet pools to provider subnet IDs
      "ingress": ["subnet-23456789"],                  // Subnets for ingress/load balancer nodes
      "workers": ["subnet-87654321", "subnet-abcdef"] // Each key can map to multiple subnets for capacity
    },
    // Optional: restrict dynamic groups to these subnet pool IDs. Omit or set empty to allow any configured subnet pool.
    "dynamic_subnet_pools": ["workers"],
    // Optional: omit shard timing fields to use server defaults.
    "request_timeout": "30s",
    "create_rate_limit": "1s",
    "health_check_interval": "60s",
    "default_drain_timeout": "5m",
    "image_refresh_interval": "6h", // How often to refresh image resolutions
    "shutdown_timeout": "10s",
    // Optional: omit garbage_collection to use server defaults.
    "garbage_collection": {
      "interval": "2m",                 // How often to backfill provider instances and look for dangling ones
      "registration_timeout": "5m",     // How long to wait for instance registration before terminating as dangling
      "deleted_record_retention": "30m" // How long to keep deleted instance records in the local database
    },
    // Optional: omit expiry to disable age-based expiry.
    "expiry": {
      "eligible_age": "504h", // 21 days
      "forced_age": "720h",   // 30 days
      "ondemand_age": "168h"  // 7 days
    },
    // Optional: shard leader election timings. Omit to use server defaults.
    "leader_election": {
      "enabled": true,
      "frequent_interval": "5s",
      "infrequent_interval": "30s",
      "leader_timeout": "15s"
    },
    // Optional: jittered delay range before exiting after fatal errors. Omit to use server defaults.
    "error_exit_jitter": {
      "min_delay": "10s",
      "max_delay": "40s"
    }
  },
  // Optional: logical backend registration targets that groups can reference by key.
  // Entries use provider-specific AWS target group, Google Cloud NEG, or tunnel listener shapes.
  "load_balancers": {
    "www": {
      "provider": "aws",
      "target_groups": [{
        "arn": "arn:aws:elasticloadbalancing:us-west-2:123412341234:targetgroup/example-www/abcdef1234567890",
        "listener_port": 443,
        "target_port": 443,
        "proxy_port": 8443
      }]
    },
    "web-tunnel": {
      "provider": "tunnel",
      "listeners": [{"target_port": 80, "proxy_port": 8080}]
    }
  },
  "tunnels": {
    "web-tunnel": {
      // Example cloudflared tunnel (replace the digest with a real pinned image digest)
      "image": "cloudflare/cloudflared@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
      "args": ["tunnel", "--config", "/run/tunnel/config.json", "run"],
      "files": {
        "/run/tunnel/config.json": {
          "kind": "json",
          "template": {
            "tunnel": "6ff42ae2-765d-4adf-8112-31c55c1551ef", // replace with a real tunnel ID
            "credentials-file": "/run/tunnel/credentials.json",
            "metrics": "127.0.0.1:2000",
            "ingress": [
              {"hostname": "www.example.com", "service": "http://127.0.0.1:8080"},
              {"service": "http_status:404"}
            ]
          }
        },
        "/run/tunnel/credentials.json": {
          // Pre-create /example-cluster/tunnel-credentials as a SecureString
          // containing the existing tunnel's credentials JSON.
          "kind": "secret",
          "source": "tunnel-credentials"
        }
      },
      "readiness_url": "http://127.0.0.1:2000/ready",
      "readiness_timeout": "30s"
    }
  },
  // Optional: protected files generated for unrelated services local to the server host.
  "server": {
    "files": {
      "service.json": {
        "kind": "storage",
        "source": "services/local.json"
      }
    }
  },
  // Optional: image lookup configuration. Omit if templates use fixed provider image IDs directly.
  "images": {
    "debian_13_arm64": {
      "provider": "aws",
      "filters": [
        {"name": "name", "values": ["debian-13-arm64-*"]},
        {"name": "virtualization-type", "values": ["hvm"]},
        {"name": "architecture", "values": ["arm64"]}
      ],
      "owners": ["136693071363"], // Debian
      "sort": "creation-date",
      "order": "desc",
      // Optional: fallback image ID if lookup fails and no cached image is available.
      "fallback": "ami-026fccd88446aa0bf"
    },
    "debian_13_amd64": {
      "provider": "aws",
      "filters": [
        {"name": "name", "values": ["debian-13-amd64-*"]},
        {"name": "virtualization-type", "values": ["hvm"]},
        {"name": "architecture", "values": ["x86_64"]}
      ],
      "owners": ["136693071363"],
      "sort": "creation-date",
      "order": "desc",
      // Optional: fallback image ID if lookup fails and no cached image is available.
      "fallback": "ami-0abcd1234efgh5678"
    }
  },
  // Optional: certificate templates for generated files.
  "certificates": {
    "kubelet.client": {
      "kind": "client",
      "cn": "system:node:{{ .Instance.ID }}",
      "organization": ["system:nodes"],
      "dns": [
        "localhost",
        "{{ .Instance.ID }}",
        "{{ .Instance.Hostname }}"
      ],
      "ip": [
        "127.0.0.1",
        "::1",
        "{{ .Instance.IP4 }}",
        "{{ .Instance.IP6 }}"
      ],
      // Optional: certificate TTL in hours. Omit to use server default.
      "ttl": 8760
    },
    "kubelet.server": {
      "kind": "server",
      "cn": "kubelet",
      "dns": [
        "localhost",
        "{{ .Instance.ID }}",
        "{{ .Instance.Hostname }}"
      ],
      "ip": [
        "127.0.0.1",
        "::1",
        "{{ .Instance.IP4 }}",
        "{{ .Instance.IP6 }}"
      ]
    },
    "kube-apiserver.server": {
      "kind": "server",
      "cn": "kubernetes",
      "dns": [
        "localhost",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local",
        "kubernetes.svc.cluster.local",
        "{{ .Instance.ID }}",
        "{{ .Instance.Hostname }}",
        "{{ .Vars.ClusterFQDN }}"
      ],
      "ip": [
        "127.0.0.1",
        "::1",
        "198.18.0.1",
        "fdc6::1",
        "{{ .Instance.IP4 }}",
        "{{ .Instance.IP6 }}"
      ]
    }
  },
  "defaults": {
    "args": {
      // Passed to the provider instance creation API. For AWS, this maps to RunInstances.
      // Nstance manages instance type, subnet, userdata, and count fields.
      "Ipv6AddressCount": 1,
      "BlockDeviceMappings": [
        {
          "DeviceName": "/dev/sda1", // root volume
          "Ebs": {
            "VolumeSize": 50, // in GB
            "VolumeType": "gp3",
            "Encrypted": true
          }
        }
      ],
      "MetadataOptions": {
        "HttpEndpoint": "enabled",      // default
        "HttpProtocolIpv6": "disabled", // default
        "HttpPutResponseHopLimit": 1,    // default
        "HttpTokens": "required",       // always require IMDSv2
        "InstanceMetadataTags": "enabled"
      },
      "PrivateDnsNameOptions": {
        "EnableResourceNameDnsARecord": true
      },
      "TagSpecifications": [
        {
          "ResourceType": "instance",
          "Tags": [
            {"Key": "InstanceId", "Value": "{{ .Instance.ID }}"},
            {"Key": "InstanceKind", "Value": "{{ .Instance.Kind }}"}
          ]
        }
      ]
    },
    "vars": {
      "ClusterSlug": "example-cluster",
      "ClusterFQDN": "example-cluster.cluster.cool",
      "Environment": "production",
      "CloudBillingID": "123412341234",
      "SSHAuthorizedKey": "",
      "TelemetryBucket": ""
    },
    // Optional: global userdata used by templates that do not define userdata.
    "userdata": {
      "content": "#!/bin/bash\necho \"default userdata for {{ .Instance.ID }}\""
    }
  },
  "templates": {
    "nat": {
      "kind": "nat",
      "arch": "arm64",
      "args": {
        "ImageId": "{{ .Image.debian_13_arm64 }}", // resolved from images config
        "SecurityGroupIds": ["sg-12341234abcd1234ab"],
        "IamInstanceProfile": {
          "Arn": "arn:aws:iam::123412341234:role/example-instance-role"
        }
      },
      "userdata": {
        "content": "#!/bin/bash\necho \"hello from {{ .Instance.ID }}\""
      },
      // template defaults:
      "size": 1,
      "instance_type": "t4g.small",
      "subnet_pool": "nat",
      "vars": {}
    },
    "knc": {
      "kind": "knc",  // 3 lowercase letters, used as prefix for instance ID (puidv7)
      "arch": "arm64", // used to validate instance type
      "files": {
        // send secret `tunnel-20250924.json` as `tunnel.json` file
        "tunnel.json": {
          "kind": "secret",
          "source": "tunnel-20250924.json"
        },
        // send a file from object storage
        "bootstrap-config.json": {
          "kind": "storage",
          "source": "files/{{ .Vars.Environment }}/bootstrap-config.json"
        },
        // send a certificate generated using a public key sent from the agent
        "kubelet.client.crt": {
          "kind": "certificate",
          "template": "kubelet.client",
          "key": {
            "source": "agent",
            "name": "kubelet.client.pub"
          }
        },
        // send templated environment file
        "instance.env": {
          "kind": "env",
          "template": {
            "INSTANCE_ID": "{{ .Instance.ID }}",
            "ENVIRONMENT": "{{ .Vars.Environment }}",
            "K8S_NODE_LABELS": "{{ .Vars.KUBELET_NODE_LABELS }}",
            "CLUSTER_FQDN": "{{ .Vars.ClusterFQDN }}"
          }
        },
        // send templated JSON configuration file
        "kubelet-config.json": {
          "kind": "json",
          "template": {
            "kind": "KubeletConfiguration",
            "apiVersion": "kubelet.config.k8s.io/v1beta1",
            "address": "{{ .Instance.IP4 }}",
            "port": 10250,
            "cgroupDriver": "systemd",
            "clusterDomain": "cluster.local",
            "nodeLabels": {
              "instance.example.com/id": "{{ .Instance.ID }}"
            }
          }
        },
        "custom.conf": {
          "kind": "string",
          "template": "instance={{ .Instance.ID }}\ncluster={{ .Vars.ClusterSlug }}\n"
        }
      },
      "args": {
        "ImageId": "{{ .Image.debian_13_arm64 }}", // resolved from images config
        "SecurityGroupIds": ["sg-23412341abcd1234ab"],
        "IamInstanceProfile": {
          "Arn": "arn:aws:iam::123412341234:role/example-instance-role"
        },
        // Example of overriding a default arg. Objects are merged recursively; non-object values replace the lower-priority value.
        "BlockDeviceMappings": [
          {
            "Ebs": {
              "VolumeSize": 100 // in GB
            }
          }
        ]
      },
      "userdata": {
        "source": "url",
        "content": "https://example.com/userdata/knc_arm64.sh"
      },
      // template defaults:
      "size": 1,
      "instance_type": "t4g.medium",
      "subnet_pool": "control-plane",
      "vars": {
        "K8SAPIHostname": "example-cluster.cluster.cool",
        "OIDCIssuer": "https://auth.example.com",
        "RegistryBucket": ""
      }
    },
    "knd": {
      "kind": "knd",
      "arch": "arm64",
      "args": {
        "ImageId": "{{ .Image.debian_13_arm64 }}"
      },
      "instance_type": "t4g.large",
      "subnet_pool": "workers"
    }
  },
  "groups": {
    "default": {
      "nat": {
        "template": "nat",
        "instance_type": "t4g.small",
        "subnet_pool": "nat" // Public service subnet pool for NAT VMs
      },
      "main": {
        "template": "knc",
        "size": 1,
        "subnet_pool": "control-plane",
        "vars": {
          "KUBELET_NODE_LABELS": "controlplane"
        }
      },
      "ingress": {
        "template": "knd",
        "subnet_pool": "ingress",
        "load_balancers": ["www", "web-tunnel"],
        "vars": {
          "KUBELET_NODE_LABELS": "traefik"
        }
      },
      "apps": {
        "template": "knd",
        "size": 2,
        "instance_type": "t4g.xlarge",
        "subnet_pool": "workers",
        "drain_timeout": "10m",
        "vars": {}
      }
    }
  },
  "nat": {
    "default": {
      "group": "nat",
      "public_addresses": [
        {"ipv4": "192.0.2.10", "allocation_id": "eipalloc-0123456789abcdef0"}
      ],
      "last_instance_grace_period": "10m",
      "instance_type_ladder": ["t4g.small", "t4g.medium", "t4g.large"],
      "scale_up_thresholds": {
        "conntrack_percent": 80,
        "cpu_percent": 80,
        "packet_drops_per_second": 1
      },
      "scale_down_thresholds": {
        "conntrack_percent": 30,
        "cpu_percent": 30,
        "packet_drops_per_second": 0
      }
    }
  }
}

{
  "provider": "aws",
  "target_groups": [{"arn": "arn:aws:elasticloadbalancing:...", "listener_port": 443, "target_port": 443, "proxy_port": 8443}]
}

{
  "provider": "google",
  "network_endpoint_groups": ["example-us-central1-a"],
  "frontends": [{"ip": "34.10.20.30", "port": 443}]
}