Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ KubeSolo supports the following command-line flags:
| `--local-storage-shared-path` | `KUBESOLO_LOCAL_STORAGE_SHARED_PATH` | Path to the shared file system for the local storage | `""` |
| `--debug` | `KUBESOLO_DEBUG` | Enable debug logging | `false` |
| `--pprof-server` | `KUBESOLO_PPROF_SERVER` | Enable pprof server for profiling | `false` |
| `--full` | `KUBESOLO_FULL` | Disable memory-saving overrides and use upstream Kubernetes defaults (recommended for CI and development) | `false` |
| `--full` | `KUBESOLO_FULL` | Deprecated: has no effect. KubeSolo always uses upstream Kubernetes defaults. Retained for backwards compatibility; will be removed in a future release | `false` |
Comment thread
stevensbkang marked this conversation as resolved.
| `--container-mode` | `KUBESOLO_CONTAINER_MODE` | Run KubeSolo inside a container, adjusting cgroups, mounts, DNS, and eviction thresholds. Auto-detected when running in a container. See [docs/configuration/container-mode.md](docs/configuration/container-mode.md) | _(auto-detected)_ |
| `--db-wal-repair` | `KUBESOLO_DB_WAL_REPAIR` | Run SQLite integrity checks on startup and repair WAL/SHM artifacts if corruption is detected | `false` |
| `--disable-ipv6` | `KUBESOLO_DISABLE_IPV6` | Disable IPv6 support for CoreDNS reverse zones and kubelet node address registration | `false` |
Expand Down
17 changes: 5 additions & 12 deletions cmd/kubesolo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ type kubesolo struct {
loadBalancer bool
localStorage bool
localStorageSharedPath string
fullMode bool
disableIPv6 bool
dbWALRepair bool
d2k bool
Expand Down Expand Up @@ -90,7 +89,6 @@ func service() (*kubesolo, error) {
loadBalancer: *flags.LoadBalancer,
localStorage: *flags.LocalStorage,
localStorageSharedPath: *flags.LocalStorageSharedPath,
fullMode: *flags.Full,
disableIPv6: *flags.DisableIPv6,
dbWALRepair: *flags.DBWALRepair,
d2k: d2kEnabled,
Expand All @@ -111,6 +109,10 @@ func main() {
os.Exit(0)
}

if *flags.Full {
log.Warn().Str("component", "kubesolo").Msg("the --full flag (KUBESOLO_FULL) is deprecated and has no effect; KubeSolo always uses upstream Kubernetes defaults")
}
Comment thread
stevensbkang marked this conversation as resolved.

if *flags.StartupTimeout > 0 {
types.DefaultRetryCount = *flags.StartupTimeout / int(types.DefaultComponentSleep.Seconds())
}
Expand Down Expand Up @@ -139,16 +141,10 @@ func (s *kubesolo) run() {
cancel()
}()

profile := "edge"
if s.fullMode {
profile = "full"
}

log.Info().
Str("version", Version).
Str("build-date", BuildDate).
Str("commit", Commit).
Str("profile", profile).
Msg("starting kubesolo...")

log.Info().Str("component", "kubesolo").Msg("ensuring all embedded dependencies are available...")
Expand Down Expand Up @@ -232,7 +228,7 @@ func (s *kubesolo) run() {
{
name: "kubeproxy",
start: func() {
kubeproxyService := kubeproxy.NewService(ctx, cancel, kubeproxyReadyCh, s.embedded.AdminKubeconfigFile, s.embedded.ContainerMode, s.embedded.FullMode)
kubeproxyService := kubeproxy.NewService(ctx, cancel, kubeproxyReadyCh, s.embedded.AdminKubeconfigFile, s.embedded.ContainerMode)
s.wg.Go(func() {
_ = kubeproxyService.Run(kubeletReadyCh)
})
Expand Down Expand Up @@ -551,9 +547,6 @@ func (s *kubesolo) bootstrap() {
// Container Mode
ContainerMode: containerMode,

// Full mode
FullMode: s.fullMode,

// IPv6
DisableIPv6: s.disableIPv6,

Expand Down
2 changes: 1 addition & 1 deletion docs/configuration/cni.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ For components like `cilium-operator` that default to multiple replicas, set the

Tested with Cilium `1.19.5` installed via the [Cilium CLI](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/).

> **Resource requirements:** Cilium is heavy. KubeSolo will not run it on a 512MB device — the Cilium agent, Envoy, and operator alone exceed that budget. Run on a beefier machine, and start KubeSolo with the `--full` flag (`KUBESOLO_FULL=true`) so the memory-saving overrides are disabled and upstream Kubernetes defaults are used, for the best outcome.
> **Resource requirements:** Cilium is heavy. KubeSolo will not run it on a 512MB device — the Cilium agent, Envoy, and operator alone exceed that budget. Run on a beefier machine for the best outcome.

### Install

Expand Down
6 changes: 3 additions & 3 deletions docs/configuration/container-mode.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ Running a Kubernetes node inside a container means the kubelet, containerd, and
| cgroup driver | `systemd` / `cgroupfs` per host | `cgroupfs`, with controller delegation set up on the root cgroup |
| Mount propagation | inherited from host | `/` remounted `rshared` so kubelet can propagate volume mounts (e.g. projected service-account tokens) into pods |
| kubelet QoS cgroups | enabled | `cgroupsPerQOS: false`, `enforceNodeAllocatable: []` — avoids the cgroupv2 "no internal processes" conflict |
| Eviction / image GC | edge or upstream thresholds | relaxed (`memory.available: 50Mi`, disk thresholds `0%`, `imageGCHighThresholdPercent: 100`) so a containerised node isn't evicted by the host's disk usage |
| Eviction / image GC | upstream thresholds | relaxed (`memory.available: 50Mi`, disk thresholds `0%`, `imageGCHighThresholdPercent: 100`) so a containerised node isn't evicted by the host's disk usage |
| Pod DNS (`resolvConf`) | host `/etc/resolv.conf` | `/dev/null`, to prevent the host's DNS config leaking into pods |
| CoreDNS upstream | `forward . /etc/resolv.conf` | `forward . 1.1.1.1 8.8.8.8` (since the node `resolv.conf` is empty) |
| CoreDNS resources | memory limit `64Mi` | memory limit removed (requests retained) to avoid OOM under a constrained container memory limit |
| kube-proxy conntrack | tuned `conntrack-max-per-core` / `conntrack-min` | both set to `0` — avoids writing to `/proc/sys/net/netfilter/nf_conntrack_max`, which is often read-only inside a container |
| kube-proxy conntrack | upstream defaults | both set to `0` — avoids writing to `/proc/sys/net/netfilter/nf_conntrack_max`, which is often read-only inside a container |

> **Container mode implies upstream defaults, not the edge profile.** The edge memory-saving overrides (`--full=false`) are skipped in container mode in favour of the adjustments above. NodeSetter is still used in place of the scheduler, as always.
> **Container mode applies the adjustments above on top of the standard upstream Kubernetes defaults.** NodeSetter is still used in place of the scheduler, as always.

---

Expand Down
2 changes: 1 addition & 1 deletion docs/installation/kubesoloctl.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ Requires a running container engine (Docker Engine / Docker Desktop).
kubesoloctl install --run-mode=container
```

KubeSolo starts in a container, the API server is published on a random localhost port, and your kubeconfig is merged and pointed at it automatically. Because container mode targets dev/CI, it always runs with upstream Kubernetes defaults (`--full`) rather than the edge memory-saving profile.
KubeSolo starts in a container, the API server is published on a random localhost port, and your kubeconfig is merged and pointed at it automatically. Container mode also adjusts cgroups, mounts, DNS, and eviction thresholds so the node comes up cleanly inside a container.

```bash
kubectl get nodes --watch
Expand Down
5 changes: 0 additions & 5 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,6 @@ PORTAINER_EDGE_ASYNC="${KUBESOLO_PORTAINER_EDGE_ASYNC:-false}"
LOAD_BALANCER="${KUBESOLO_LOAD_BALANCER:-true}"
LOCAL_STORAGE="${KUBESOLO_LOCAL_STORAGE:-true}"
LOCAL_STORAGE_SHARED_PATH="${KUBESOLO_LOCAL_STORAGE_SHARED_PATH:-}"
FULL="${KUBESOLO_FULL:-false}"
DB_WAL_REPAIR="${KUBESOLO_DB_WAL_REPAIR:-false}"
DISABLE_IPV6="${KUBESOLO_DISABLE_IPV6:-false}"
STARTUP_TIMEOUT="${KUBESOLO_STARTUP_TIMEOUT:-600}"
Expand Down Expand Up @@ -1317,10 +1316,6 @@ if [ -n "$LOCAL_STORAGE_SHARED_PATH" ]; then
CMD_ARGS="$CMD_ARGS --local-storage-shared-path=$LOCAL_STORAGE_SHARED_PATH"
fi

if [ "$FULL" = "true" ]; then
CMD_ARGS="$CMD_ARGS --full=true"
fi

if [ "$DB_WAL_REPAIR" = "true" ]; then
CMD_ARGS="$CMD_ARGS --db-wal-repair=true"
fi
Expand Down
2 changes: 1 addition & 1 deletion internal/config/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ var (
Debug = Application.Flag("debug", "Enable debug logging. Defaults to false.").Envar("KUBESOLO_DEBUG").Default("false").Bool()
PprofServer = Application.Flag("pprof-server", "Enable pprof server. Defaults to false.").Envar("KUBESOLO_PPROF_SERVER").Default("false").Bool()
ContainerMode = Application.Flag("container-mode", "Run in container mode with cgroupfs driver and relaxed eviction thresholds. Auto-detected when running inside a container.").Envar("KUBESOLO_CONTAINER_MODE").Bool()
Full = Application.Flag("full", "Disable memory-saving overrides and use upstream Kubernetes defaults. Kubesolo still uses NodeSetter in favour of the scheduler. Recommended for CI and developer environments where memory is not constrained. Leave unset for edge deployments.").Envar("KUBESOLO_FULL").Default("false").Bool()
Full = Application.Flag("full", "Deprecated: has no effect. KubeSolo always uses upstream Kubernetes defaults. Retained for backwards compatibility and will be removed in a future release.").Envar("KUBESOLO_FULL").Default("false").Bool()
Comment thread
stevensbkang marked this conversation as resolved.
DBWALRepair = Application.Flag("db-wal-repair", "On startup, run an integrity check against the SQLite database and remove WAL artefacts (state.db-wal, state.db-shm) if corruption is detected. Recovers from unclean shutdowns caused by power loss. Defaults to false.").Envar("KUBESOLO_DB_WAL_REPAIR").Default("false").Bool()
DisableIPv6 = Application.Flag("disable-ipv6", "Disable IPv6 support. When set, CoreDNS will not serve ip6.arpa reverse zones and kubelet will register with an explicit IPv4 node address. Defaults to false.").Envar("KUBESOLO_DISABLE_IPV6").Default("false").Bool()
StartupTimeout = Application.Flag("startup-timeout", "Maximum time in seconds to wait for each component to pass its health check during startup. Increase on slow storage such as SD cards. Defaults to 600.").Envar("KUBESOLO_STARTUP_TIMEOUT").Default("600").Int()
Expand Down
28 changes: 0 additions & 28 deletions pkg/kubernetes/apiserver/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,33 +42,5 @@ func (s *service) configureAPIServerFlags(command *cobra.Command) error {
// feature gates - disable SizeBasedListCostEstimate to suppress "Error getting keys" messages
_ = flags.Set("feature-gates", "SizeBasedListCostEstimate=false")

// Edge-optimised overrides — only applied when not in full mode.
// When full mode is enabled, upstream Kubernetes defaults are used instead.
if !s.fullMode {
// etcd metric collection
_ = flags.Set("etcd-count-metric-poll-period", "0")
_ = flags.Set("etcd-db-metric-poll-interval", "0")

// request throttling and timeouts
_ = flags.Set("max-requests-inflight", "2000")
_ = flags.Set("max-mutating-requests-inflight", "1000")
_ = flags.Set("min-request-timeout", "180")
_ = flags.Set("request-timeout", "900s")
_ = flags.Set("kubelet-timeout", "30s")

// diagnostics
_ = flags.Set("profiling", "false")

// admission control
_ = flags.Set("enable-admission-plugins", "NodeRestriction,ServiceAccount,ValidatingAdmissionWebhook,MutatingAdmissionWebhook,DefaultStorageClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionPolicy,MutatingAdmissionPolicy")
_ = flags.Set("disable-admission-plugins", "RuntimeClass,PodSecurity,ClusterTrustBundleAttest,DefaultIngressClass,TaintNodesByCondition,DefaultTolerationSeconds,StorageObjectInUseProtection,PersistentVolumeClaimResize,ResourceQuota,LimitRanger,Priority")

// audit logging
_ = flags.Set("audit-log-path", "-")
_ = flags.Set("audit-log-maxage", "0")
_ = flags.Set("audit-log-maxbackup", "0")
_ = flags.Set("audit-log-maxsize", "0")
}

return nil
}
2 changes: 0 additions & 2 deletions pkg/kubernetes/apiserver/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ type service struct {
requestHeaderCAFile string
requestHeaderClientCert string
requestHeaderClientKey string
fullMode bool
kubeSoloWebhook *webhook.Service
}

Expand All @@ -52,7 +51,6 @@ func NewService(ctx context.Context, cancel context.CancelFunc, apiServerReady c
requestHeaderCAFile: embedded.RequestHeaderCerts.CACert,
requestHeaderClientCert: embedded.RequestHeaderCerts.ClientCert,
requestHeaderClientKey: embedded.RequestHeaderCerts.ClientKey,
fullMode: embedded.FullMode,
kubeSoloWebhook: webhook.NewService(nodeName, embedded.NodeIP, embedded.PKIDir, embedded.AdminKubeconfigFile, embedded.LoadBalancer),
}
}
46 changes: 0 additions & 46 deletions pkg/kubernetes/controller/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,50 +21,4 @@ func (s *service) configureControllerManagerFlags(command *cobra.Command) {
_ = flags.Set("tls-private-key-file", s.controllerManagerKeyFile)
_ = flags.Set("leader-elect", "false")
_ = flags.Set("use-service-account-credentials", "true")

// Edge-optimised overrides — only applied when not in full mode.
// When full mode is enabled, upstream Kubernetes defaults are used instead.
if !s.fullMode {
// controllers
_ = flags.Set("controllers", "deployment,replicaset,service,serviceaccount,namespace,attachdetach,endpoint,daemonset,statefulset,root-ca-certificate-publisher-controller,serviceaccount-token-controller,node-ipam-controller,endpointslice-controller,persistentvolume-binder-controller,job-controller,cronjob-controller,garbage-collector-controller,disruption,csrsigning,clusterrole-aggregation")

_ = flags.Set("profiling", "false")
_ = flags.Set("terminated-pod-gc-threshold", "20")
_ = flags.Set("large-cluster-size-threshold", "10")
_ = flags.Set("unhealthy-zone-threshold", "0.7")

// sync settings
_ = flags.Set("concurrent-deployment-syncs", "2")
_ = flags.Set("concurrent-replicaset-syncs", "2")
_ = flags.Set("concurrent-job-syncs", "2")
_ = flags.Set("concurrent-endpoint-syncs", "2")
_ = flags.Set("concurrent-service-endpoint-syncs", "2")
_ = flags.Set("concurrent-gc-syncs", "2")
_ = flags.Set("concurrent-namespace-syncs", "2")
_ = flags.Set("concurrent-cron-job-syncs", "2")
_ = flags.Set("concurrent-horizontal-pod-autoscaler-syncs", "2")
_ = flags.Set("concurrent-rc-syncs", "2")
_ = flags.Set("concurrent-resource-quota-syncs", "2")
_ = flags.Set("concurrent-service-syncs", "2")
_ = flags.Set("concurrent-serviceaccount-token-syncs", "2")
_ = flags.Set("concurrent-statefulset-syncs", "2")
_ = flags.Set("concurrent-ttl-after-finished-syncs", "2")
_ = flags.Set("concurrent-ephemeralvolume-syncs", "2")
_ = flags.Set("concurrent-validating-admission-policy-status-syncs", "2")
_ = flags.Set("mirroring-concurrent-service-endpoint-syncs", "2")

// sync period
_ = flags.Set("horizontal-pod-autoscaler-sync-period", "60s")
_ = flags.Set("node-monitor-period", "60s")
_ = flags.Set("pvclaimbinder-sync-period", "120s")
_ = flags.Set("resource-quota-sync-period", "15m")
_ = flags.Set("namespace-sync-period", "15m")
_ = flags.Set("route-reconciliation-period", "60s")
_ = flags.Set("attach-detach-reconcile-sync-period", "10m")
_ = flags.Set("node-monitor-grace-period", "300s")

// api server interactions
_ = flags.Set("kube-api-qps", "50")
_ = flags.Set("kube-api-burst", "100")
}
}
2 changes: 0 additions & 2 deletions pkg/kubernetes/controller/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ type service struct {
caFile string
adminKubeconfigFile string
serviceAccountKeyFile string
fullMode bool
}

// NewService creates a new controller service
Expand All @@ -34,6 +33,5 @@ func NewService(ctx context.Context, cancel context.CancelFunc, controllerReady
caFile: embedded.CACerts.Cert,
adminKubeconfigFile: embedded.AdminKubeconfigFile,
serviceAccountKeyFile: embedded.ServiceAccountKeyFile,
fullMode: embedded.FullMode,
}
}
31 changes: 0 additions & 31 deletions pkg/kubernetes/kubelet/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,36 +111,5 @@ func (s *service) generateKubeletConfig() map[string]any {
return config
}

// Edge-optimised overrides — only applied when not in full mode.
// When full mode is enabled, upstream Kubernetes defaults are used instead.
if !s.fullMode {
config["enableProfilingHandler"] = false
config["enableDebugFlagsHandler"] = false
config["streamingConnectionIdleTimeout"] = "1h0s"
config["syncFrequency"] = "5m0s"
config["fileCheckFrequency"] = "2m0s"
config["httpCheckFrequency"] = "2m0s"
config["nodeStatusUpdateFrequency"] = "60s"
config["nodeStatusReportFrequency"] = "15m0s"
config["volumeStatsAggPeriod"] = "5m0s"
config["imageMinimumGCAge"] = "10m0s"
config["imageMaximumGCAge"] = "0s"
config["imageGCHighThresholdPercent"] = 95
config["runtimeRequestTimeout"] = "60s"
config["cpuManagerReconcilePeriod"] = "60s"
config["kubeAPIQPS"] = 10
config["kubeAPIBurst"] = 20
config["eventRecordQPS"] = 5
config["eventBurst"] = 10
config["containerLogMaxSize"] = "512Ki"
config["maxPods"] = 20
config["evictionHard"] = map[string]string{
"memory.available": "75Mi",
"nodefs.available": "50Mi",
}
config["systemReserved"] = map[string]string{"memory": "25Mi"}
config["kubeReserved"] = map[string]string{"memory": "25Mi"}
}

return config
}
37 changes: 9 additions & 28 deletions pkg/kubernetes/kubelet/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestGenerateKubeletConfig_Common(t *testing.T) {
}

func TestGenerateKubeletConfig_ContainerMode(t *testing.T) {
s := &service{kubeletDir: t.TempDir(), containerMode: true, fullMode: false}
s := &service{kubeletDir: t.TempDir(), containerMode: true}
cfg := s.generateKubeletConfig()

// In container mode the host resolv.conf is never consulted.
Expand All @@ -56,38 +56,19 @@ func TestGenerateKubeletConfig_ContainerMode(t *testing.T) {
assert.False(t, hasMaxPods, "container mode must not set edge-mode maxPods")
}

func TestGenerateKubeletConfig_EdgeMode(t *testing.T) {
s := &service{kubeletDir: t.TempDir(), containerMode: false, fullMode: false}
func TestGenerateKubeletConfig_UpstreamDefaults(t *testing.T) {
s := &service{kubeletDir: t.TempDir(), containerMode: false}
cfg := s.generateKubeletConfig()

assert.Equal(t, 20, cfg["maxPods"])
assert.Equal(t, false, cfg["enableProfilingHandler"])
assert.Equal(t, 95, cfg["imageGCHighThresholdPercent"])

evict, ok := cfg["evictionHard"].(map[string]string)
require.True(t, ok)
assert.Equal(t, "75Mi", evict["memory.available"])

sysReserved, ok := cfg["systemReserved"].(map[string]string)
require.True(t, ok)
assert.Equal(t, "25Mi", sysReserved["memory"])

// Edge mode is not container mode — the QoS-disabling key must be absent.
_, hasCgroupsPerQOS := cfg["cgroupsPerQOS"]
assert.False(t, hasCgroupsPerQOS)
}

func TestGenerateKubeletConfig_FullMode(t *testing.T) {
s := &service{kubeletDir: t.TempDir(), containerMode: false, fullMode: true}
cfg := s.generateKubeletConfig()

// Full mode uses upstream defaults — neither edge nor container overrides apply.
for _, k := range []string{"maxPods", "enableProfilingHandler", "cgroupsPerQOS", "evictionHard"} {
// Outside container mode, KubeSolo uses upstream Kubernetes defaults — none of
// the former edge overrides or the container-mode QoS keys are set.
for _, k := range []string{"maxPods", "enableProfilingHandler", "imageGCHighThresholdPercent", "evictionHard", "systemReserved", "kubeReserved", "cgroupsPerQOS"} {
_, ok := cfg[k]
assert.Falsef(t, ok, "full mode must not set %q", k)
assert.Falsef(t, ok, "upstream defaults must not set %q", k)
}

// Baseline secure-default keys still apply in full mode.
// Baseline secure-default keys still apply.
assert.Equal(t, 0, cfg["readOnlyPort"])
assert.Equal(t, false, cfg["failSwapOn"])
assert.Equal(t, true, cfg["rotateCertificates"])
}
2 changes: 0 additions & 2 deletions pkg/kubernetes/kubelet/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ type service struct {
kubeletCertPath string
adminKubeconfig string
containerMode bool
fullMode bool
disableIPv6 bool
}

Expand All @@ -53,7 +52,6 @@ func NewService(ctx context.Context, cancel context.CancelFunc, kubeletReady cha
nodeName: system.GetHostname(),
adminKubeconfig: embedded.AdminKubeconfigFile,
containerMode: embedded.ContainerMode,
fullMode: embedded.FullMode,
disableIPv6: embedded.DisableIPv6,
}
}
5 changes: 0 additions & 5 deletions pkg/kubernetes/kubeproxy/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,6 @@ func (s *service) configureKubeProxyFlags(command *cobra.Command) {
// which may be read-only depending on the container runtime.
_ = flags.Set("conntrack-max-per-core", "0")
_ = flags.Set("conntrack-min", "0")
} else if !s.fullMode {
_ = flags.Set("profiling", "false")
_ = flags.Set("conntrack-max-per-core", "1024")
_ = flags.Set("conntrack-min", "1024")
_ = flags.Set("min-sync-period", "10s")
}

if proxyMode == "iptables" {
Expand Down
Loading
Loading