diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 92546e3f9c90..a749321f646f 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -18,6 +18,7 @@ package cmd import ( "fmt" + "net" "runtime" "strings" "time" @@ -59,6 +60,7 @@ const ( kubernetesVersion = "kubernetes-version" noKubernetes = "no-kubernetes" hostOnlyCIDR = "host-only-cidr" + hostOnlyCIDRv6 = "host-only-cidr-v6" containerRuntime = "container-runtime" criSocket = "cri-socket" networkPlugin = "network-plugin" // deprecated, use --cni instead @@ -82,6 +84,8 @@ const ( apiServerPort = "apiserver-port" dnsDomain = "dns-domain" serviceCIDR = "service-cluster-ip-range" + serviceCIDRv6 = "service-cluster-ip-range-v6" + ipFamily = "ip-family" imageRepository = "image-repository" imageMirrorCountry = "image-mirror-country" mountString = "mount-string" @@ -144,8 +148,12 @@ const ( socketVMnetClientPath = "socket-vmnet-client-path" socketVMnetPath = "socket-vmnet-path" staticIP = "static-ip" + staticIPv6 = "static-ipv6" gpus = "gpus" autoPauseInterval = "auto-pause-interval" + subnetv6 = "subnet-v6" + podCIDR = "pod-cidr" + podCIDRv6 = "pod-cidr-v6" ) var ( @@ -209,6 +217,7 @@ func initMinikubeFlags() { startCmd.Flags().Bool(disableMetrics, false, "If set, disables metrics reporting (CPU and memory usage), this can improve CPU usage. Defaults to false.") startCmd.Flags().Bool(disableCoreDNSLog, false, "If set, disable CoreDNS verbose logging. Defaults to false.") startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)") + startCmd.Flags().String(staticIPv6, "", "Set a static IPv6 address for the minikube cluster, for example fd00::100 (Docker and Podman drivers only)") startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your GPUs. Options include: [all,nvidia,amd] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") } @@ -260,6 +269,7 @@ func initDriverFlags() { // virtualbox startCmd.Flags().String(hostOnlyCIDR, "192.168.59.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)") + startCmd.Flags().String(hostOnlyCIDRv6, "fd00::1/64", "The IPv6 CIDR to be used for the minikube VM (virtualbox driver only)") startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox driver only)") startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox driver only)") startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox driver only)") @@ -282,6 +292,10 @@ func initDriverFlags() { startCmd.Flags().String(listenAddress, "", "IP Address to use to expose ports (docker and podman driver only)") startCmd.Flags().StringSlice(ports, []string{}, "List of ports that should be exposed (docker and podman driver only)") startCmd.Flags().String(subnet, "", "Subnet to be used on kic cluster. If left empty, minikube will choose subnet address, beginning from 192.168.49.0. (docker and podman driver only)") + startCmd.Flags().String(subnetv6, "", "IPv6 subnet (CIDR) for the Docker/Podman network. If empty, minikube picks an internal ULA. (docker and podman driver only)") + startCmd.Flags().String(podCIDR, "", "IPv4 CIDR to use for pod IPs (bridge CNI).") + startCmd.Flags().String(podCIDRv6, "", "IPv6 CIDR to use for pod IPs (bridge CNI).") + // qemu startCmd.Flags().String(qemuFirmwarePath, "", "Path to the qemu firmware file. Defaults: For Linux, the default firmware location. For macOS, the brew installation location. For Windows, C:\\Program Files\\qemu\\share") @@ -293,7 +307,9 @@ func initNetworkingFlags() { startCmd.Flags().StringSliceVar(®istryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon") startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers") startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.") - startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The IPv4 CIDR to be used for service cluster IPs.") + startCmd.Flags().String(serviceCIDRv6, constants.DefaultServiceCIDRv6, "The IPv6 CIDR to be used for service cluster IPs.") + startCmd.Flags().String(ipFamily, "ipv4", "Cluster IP family mode: one of 'ipv4' (default), 'ipv6', or 'dual'.") startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)") startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)") @@ -494,6 +510,94 @@ func getNetwork(driverName string) string { return n } +// normalizeAndValidateIPFamily sets defaults, validates CIDRs, and guards Desktop vs Linux daemon for v6. +func normalizeAndValidateIPFamily(cc *config.ClusterConfig) { + fam := strings.ToLower(strings.TrimSpace(cc.KubernetesConfig.IPFamily)) + switch fam { + case "", "ipv4", "ipv6", "dual": + // ok + default: + exit.Message(reason.Usage, "Invalid --ip-family {{.fam}}. Must be one of: ipv4, ipv6, dual.", out.V{"fam": cc.KubernetesConfig.IPFamily}) + } + if fam == "" { + fam = "ipv4" + cc.KubernetesConfig.IPFamily = fam + } + // default v6 CIDRs if needed + if fam != "ipv4" { + if cc.KubernetesConfig.ServiceCIDRv6 == "" { + cc.KubernetesConfig.ServiceCIDRv6 = constants.DefaultServiceCIDRv6 + } + if cc.KubernetesConfig.PodCIDRv6 == "" { + cc.KubernetesConfig.PodCIDRv6 = constants.DefaultPodCIDRv6 + } + } + // defaults so dual has both sides unless the user overrides + if fam != "ipv6" && cc.KubernetesConfig.PodCIDR == "" { + cc.KubernetesConfig.PodCIDR = cni.DefaultPodCIDR + } + + if fam != "ipv4" && cc.KubernetesConfig.PodCIDRv6 == "" { + cc.KubernetesConfig.PodCIDRv6 = constants.DefaultPodCIDRv6 + } + + // basic CIDR sanity + if cidr := cc.Subnetv6; cidr != "" { + if _, _, err := net.ParseCIDR(cidr); err != nil { + exit.Message(reason.Usage, "--subnet-v6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err}) + } + } + if cidr := cc.KubernetesConfig.ServiceCIDRv6; cidr != "" { + if _, _, err := net.ParseCIDR(cidr); err != nil { + exit.Message(reason.Usage, "--service-cluster-ip-range-v6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err}) + } + } + if cidr := cc.KubernetesConfig.PodCIDRv6; cidr != "" { + if _, _, err := net.ParseCIDR(cidr); err != nil { + exit.Message(reason.Usage, "PodCIDRv6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err}) + } + } + + if s := cc.KubernetesConfig.PodCIDR; s != "" { + if _, _, err := net.ParseCIDR(s); err != nil { + exit.Message(reason.Usage, "--pod-cidr must be a valid IPv4 CIDR: {{.e}}", out.V{"e": err}) + } + } + + if s := cc.KubernetesConfig.PodCIDRv6; s != "" { + if _, _, err := net.ParseCIDR(s); err != nil { + exit.Message(reason.Usage, "--pod-cidr-v6 must be a valid IPv6 CIDR: {{.e}}", out.V{"e": err}) + } + } + + // Docker driver guardrails: Linux daemon + IPv6 must be enabled + if driver.IsDocker(cc.Driver) && fam != "ipv4" { + // Desktop vs Linux daemon hint (we can't reliably detect IPv6 enabled here) + si, err := oci.CachedDaemonInfo(cc.Driver) + if err != nil { + si, err = oci.DaemonInfo(cc.Driver) + if err != nil { + exit.Message(reason.Usage, "Failed to query Docker daemon info: {{.e}}", out.V{"e": err}) + } + } + // On non-Linux hosts we assume Docker Desktop; on Linux it's a native Engine + // unless DockerOS explicitly says "Docker Desktop". + isLinuxDaemon := runtime.GOOS == "linux" && si.DockerOS != "Docker Desktop" + if !isLinuxDaemon { + if fam == "ipv6" { + exit.Message(reason.Usage, + "IPv6 clusters require a Linux Docker daemon (Desktop is not supported). "+ + "Use a Linux/WSL2 daemon or set --ip-family=ipv4.") + } + out.WarningT("Dual-stack on Docker Desktop may be limited. For full IPv6 support, use a Linux Docker daemon.") + } + + // Friendly reminder about enabling daemon IPv6 (actual failure will occur during network create otherwise) + out.Styled(style.Tip, + "If Docker daemon IPv6 is disabled, enable it in /etc/docker/daemon.json and restart:\n {\"ipv6\": true, \"fixed-cidr-v6\": \"fd00:55:66::/64\"}") + } +} + func validateQemuNetwork(n string) string { switch n { case "socket_vmnet": @@ -576,6 +680,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str KicBaseImage: viper.GetString(kicBaseImage), Network: getNetwork(drvName), Subnet: viper.GetString(subnet), + Subnetv6: viper.GetString(subnetv6), Memory: getMemorySize(cmd, drvName), CPUs: getCPUCount(drvName), DiskSize: getDiskSize(), @@ -590,6 +695,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str InsecureRegistry: insecureRegistry, RegistryMirror: registryMirror, HostOnlyCIDR: viper.GetString(hostOnlyCIDR), + HostOnlyCIDRv6: viper.GetString(hostOnlyCIDRv6), HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch), HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch), HypervExternalAdapter: viper.GetString(hypervExternalAdapter), @@ -631,6 +737,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str SocketVMnetClientPath: detect.SocketVMNetClientPath(), SocketVMnetPath: detect.SocketVMNetPath(), StaticIP: viper.GetString(staticIP), + StaticIPv6: viper.GetString(staticIPv6), KubernetesConfig: config.KubernetesConfig{ KubernetesVersion: k8sVersion, ClusterName: ClusterFlagValue(), @@ -644,6 +751,10 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str CRISocket: viper.GetString(criSocket), NetworkPlugin: chosenNetworkPlugin, ServiceCIDR: viper.GetString(serviceCIDR), + ServiceCIDRv6: viper.GetString(serviceCIDRv6), + PodCIDR: viper.GetString(podCIDR), + PodCIDRv6: viper.GetString(podCIDRv6), + IPFamily: viper.GetString(ipFamily), ImageRepository: getRepository(cmd, k8sVersion), ExtraOptions: getExtraOptions(), ShouldLoadCachedImages: viper.GetBool(cacheImages), @@ -697,6 +808,8 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str } } + normalizeAndValidateIPFamily(&cc) + return cc } @@ -831,11 +944,15 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.MinikubeISO, isoURL) updateStringFromFlag(cmd, &cc.KicBaseImage, kicBaseImage) updateStringFromFlag(cmd, &cc.Network, network) + updateStringFromFlag(cmd, &cc.Subnetv6, subnetv6) + updateStringFromFlag(cmd, &cc.KubernetesConfig.PodCIDR, podCIDR) + updateStringFromFlag(cmd, &cc.KubernetesConfig.PodCIDRv6, podCIDRv6) updateStringFromFlag(cmd, &cc.HyperkitVpnKitSock, vpnkitSock) updateStringSliceFromFlag(cmd, &cc.HyperkitVSockPorts, vsockPorts) updateStringSliceFromFlag(cmd, &cc.NFSShare, nfsShare) updateStringFromFlag(cmd, &cc.NFSSharesRoot, nfsSharesRoot) - updateStringFromFlag(cmd, &cc.HostOnlyCIDR, hostOnlyCIDR) + updateStringFromFlag(cmd, &cc.HostOnlyCIDR, hostOnlyCIDR) + updateStringFromFlag(cmd, &cc.HostOnlyCIDRv6, hostOnlyCIDRv6) updateStringFromFlag(cmd, &cc.HypervVirtualSwitch, hypervVirtualSwitch) updateBoolFromFlag(cmd, &cc.HypervUseExternalSwitch, hypervUseExternalSwitch) updateStringFromFlag(cmd, &cc.HypervExternalAdapter, hypervExternalAdapter) @@ -853,6 +970,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateDurationFromFlag(cmd, &cc.StartHostTimeout, waitTimeout) updateStringSliceFromFlag(cmd, &cc.ExposedPorts, ports) updateStringFromFlag(cmd, &cc.SSHIPAddress, sshIPAddress) + updateStringFromFlag(cmd, &cc.StaticIPv6, staticIPv6) updateStringFromFlag(cmd, &cc.SSHUser, sshSSHUser) updateStringFromFlag(cmd, &cc.SSHKey, sshSSHKey) updateIntFromFlag(cmd, &cc.SSHPort, sshSSHPort) @@ -864,7 +982,9 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.KubernetesConfig.ContainerRuntime, containerRuntime) updateStringFromFlag(cmd, &cc.KubernetesConfig.CRISocket, criSocket) updateStringFromFlag(cmd, &cc.KubernetesConfig.NetworkPlugin, networkPlugin) - updateStringFromFlag(cmd, &cc.KubernetesConfig.ServiceCIDR, serviceCIDR) + updateStringFromFlag(cmd, &cc.KubernetesConfig.ServiceCIDR, serviceCIDR) + updateStringFromFlag(cmd, &cc.KubernetesConfig.ServiceCIDRv6, serviceCIDRv6) + updateStringFromFlag(cmd, &cc.KubernetesConfig.IPFamily, ipFamily) updateBoolFromFlag(cmd, &cc.KubernetesConfig.ShouldLoadCachedImages, cacheImages) updateDurationFromFlag(cmd, &cc.CertExpiration, certExpiration) updateStringFromFlag(cmd, &cc.MountString, mountString) @@ -921,7 +1041,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC if cc.ScheduledStop != nil && time.Until(time.Unix(cc.ScheduledStop.InitiationTime, 0).Add(cc.ScheduledStop.Duration)) <= 0 { cc.ScheduledStop = nil } - + normalizeAndValidateIPFamily(&cc) return cc } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index f8e35de5f51d..a98ffa4a6f3a 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -89,6 +89,8 @@ func (d *Driver) Create() error { OCIBinary: d.NodeConfig.OCIBinary, APIServerPort: d.NodeConfig.APIServerPort, GPUs: d.NodeConfig.GPUs, + IPFamily: strings.ToLower(d.NodeConfig.IPFamily), + IPv6: d.NodeConfig.StaticIPv6, } if params.Memory != "0" { params.Memory += "mb" @@ -99,40 +101,93 @@ func (d *Driver) Create() error { networkName = d.NodeConfig.ClusterName } staticIP := d.NodeConfig.StaticIP - if gateway, err := oci.CreateNetwork(d.OCIBinary, networkName, d.NodeConfig.Subnet, staticIP); err != nil { + // NEW: create network with IPv6/dual awareness + gateway, err := oci.CreateNetworkWithIPFamily( + d.OCIBinary, + networkName, + d.NodeConfig.Subnet, + d.NodeConfig.Subnetv6, // NEW + staticIP, + d.NodeConfig.StaticIPv6, // NEW + params.IPFamily, // NEW + ) + if err != nil { msg := "Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}" args := out.V{"error": err} if staticIP != "" { exit.Message(reason.IfDedicatedNetwork, msg, args) } out.WarningT(msg, args) - } else if gateway != nil && staticIP != "" { - params.Network = networkName - params.IP = staticIP - } else if gateway != nil { - params.Network = networkName - ip := gateway.To4() - // calculate the container IP based on guessing the machine index - index := driver.IndexFromMachineName(d.NodeConfig.MachineName) - if int(ip[3])+index > 253 { // reserve last client ip address for multi-control-plane loadbalancer vip address in ha cluster - return fmt.Errorf("too many machines to calculate an IP") - } - ip[3] += byte(index) - klog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) - params.IP = ip.String() } + // Always attach to the created user network (even if gateway is nil in IPv6-only) + params.Network = networkName + + // Now decide static IPs per family + switch params.IPFamily { + case "ipv6": + if d.NodeConfig.StaticIPv6 != "" { + params.IPv6 = d.NodeConfig.StaticIPv6 + } + case "dual": + // IPv4 part (only if Docker reported a v4 gateway) + if g4 := gateway.To4(); g4 != nil { + if staticIP != "" { + params.IP = staticIP + } else { + ip := make(net.IP, len(g4)) + copy(ip, g4) + index := driver.IndexFromMachineName(d.NodeConfig.MachineName) + if int(ip[3])+index > 253 { + return fmt.Errorf("too many machines to calculate an IPv4") + } + ip[3] += byte(index) + klog.Infof("calculated static IPv4 %q for the %q container", ip.String(), d.NodeConfig.MachineName) + params.IP = ip.String() + } + } + if d.NodeConfig.StaticIPv6 != "" { + params.IPv6 = d.NodeConfig.StaticIPv6 + } + default: // ipv4 + if staticIP != "" { + params.IP = staticIP + } else if gateway != nil { + if g4 := gateway.To4(); g4 != nil { + ip := make(net.IP, len(g4)) + copy(ip, g4) + index := driver.IndexFromMachineName(d.NodeConfig.MachineName) + if int(ip[3])+index > 253 { + return fmt.Errorf("too many machines to calculate an IP") + } + ip[3] += byte(index) + klog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) + params.IP = ip.String() + } + } + } drv := d.DriverName() - + // Default listen address: v4 localhost for ipv4, v6 localhost for ipv6-only listAddr := oci.DefaultBindIPV4 + // IPv6-only clusters must publish on IPv6 loopback so the host can reach them + if params.IPFamily == "ipv6" { + listAddr = "::1" + } + if d.NodeConfig.ListenAddress != "" && d.NodeConfig.ListenAddress != listAddr { out.Step(style.Tip, "minikube is not meant for production use. You are opening non-local traffic") out.WarningT("Listening to {{.listenAddr}}. This is not recommended and can cause a security vulnerability. Use at your own risk", out.V{"listenAddr": d.NodeConfig.ListenAddress}) listAddr = d.NodeConfig.ListenAddress } else if oci.IsExternalDaemonHost(drv) { - out.WarningT("Listening to 0.0.0.0 on external docker host {{.host}}. Please be advised", - out.V{"host": oci.DaemonHost(drv)}) - listAddr = "0.0.0.0" + if params.IPFamily == "ipv6" { + out.WarningT("Listening to :: on external docker host {{.host}}. Please be advised", + out.V{"host": oci.DaemonHost(drv)}) + listAddr = "::" + } else { + out.WarningT("Listening to 0.0.0.0 on external docker host {{.host}}. Please be advised", + out.V{"host": oci.DaemonHost(drv)}) + listAddr = "0.0.0.0" + } } // control plane specific options @@ -293,18 +348,38 @@ func (d *Driver) DriverName() string { // GetIP returns an IP or hostname that this host is available at func (d *Driver) GetIP() (string, error) { - ip, _, err := oci.ContainerIPs(d.OCIBinary, d.MachineName) - return ip, err + ip4, ip6, err := oci.ContainerIPs(d.OCIBinary, d.MachineName) + if err != nil { + return "", err + } + switch strings.ToLower(d.NodeConfig.IPFamily) { + case "ipv6": + if ip6 != "" { + return ip6, nil + } + } + // default / dual prefers IPv4 for backward compat + return ip4, nil } // GetExternalIP returns an IP which is accessible from outside func (d *Driver) GetExternalIP() (string, error) { - return oci.DaemonHost(d.DriverName()), nil + host := oci.DaemonHost(d.DriverName()) + // For local daemons and IPv6-only clusters, ports are published on ::1 + if strings.ToLower(d.NodeConfig.IPFamily) == "ipv6" && !oci.IsExternalDaemonHost(d.DriverName()) { + return "::1", nil + } + return host, nil } // GetSSHHostname returns hostname for use with ssh func (d *Driver) GetSSHHostname() (string, error) { - return oci.DaemonHost(d.DriverName()), nil + host := oci.DaemonHost(d.DriverName()) + // For local daemons and IPv6-only clusters, ports are published on ::1 + if strings.ToLower(d.NodeConfig.IPFamily) == "ipv6" && !oci.IsExternalDaemonHost(d.DriverName()) { + return "::1", nil + } + return host, nil } // GetSSHPort returns port for use with ssh diff --git a/pkg/drivers/kic/oci/network_create.go b/pkg/drivers/kic/oci/network_create.go index cd38dc785c30..915e795c3eb6 100644 --- a/pkg/drivers/kic/oci/network_create.go +++ b/pkg/drivers/kic/oci/network_create.go @@ -36,6 +36,7 @@ import ( // defaultFirstSubnetAddr is a first subnet to be used on first kic cluster // it is one octet more than the one used by KVM to avoid possible conflict const defaultFirstSubnetAddr = "192.168.49.0" +const defaultFirstSubnetAddrv6 = "fd00::/64" // name of the default bridge network, used to lookup the MTU (see #9528) const dockerDefaultBridge = "bridge" @@ -63,14 +64,30 @@ func firstSubnetAddr(subnet string) string { return subnet } +func firstSubnetAddrv6(subnet string) string { + if subnet == "" { + return defaultFirstSubnetAddrv6 + } + return subnet +} + // CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster func CreateNetwork(ociBin, networkName, subnet, staticIP string) (net.IP, error) { - defaultBridgeName := defaultBridgeName(ociBin) - if networkName == defaultBridgeName { + return CreateNetworkWithIPFamily(ociBin, networkName, subnet, "", staticIP, "", "ipv4") +} + +func CreateNetworkWithIPFamily(ociBin, networkName, subnet, subnetv6, staticIP, staticIPv6, ipFamily string) (net.IP, error) { + bridgeName := defaultBridgeName(ociBin) + if networkName == bridgeName { klog.Infof("skipping creating network since default network %s was specified", networkName) return nil, nil } + // For IPv6-only or dual-stack networks, use the v6/dual creator + if ipFamily == "ipv6" || ipFamily == "dual" { + return createV6OrDualNetwork(ociBin, networkName, subnet, subnetv6, ipFamily) + } + // check if the network already exists info, err := containerNetworkInspect(ociBin, networkName) if err == nil { @@ -80,9 +97,9 @@ func CreateNetwork(ociBin, networkName, subnet, staticIP string) (net.IP, error) // will try to get MTU from the docker network to avoid issue with systems with exotic MTU settings. // related issue #9528 - info, err = containerNetworkInspect(ociBin, defaultBridgeName) + info, err = containerNetworkInspect(ociBin, bridgeName) if err != nil { - klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, defaultBridgeName, err) + klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, bridgeName, err) } tries := 20 @@ -119,6 +136,59 @@ func CreateNetwork(ociBin, networkName, subnet, staticIP string) (net.IP, error) return info.gateway, fmt.Errorf("failed to create %s network %s: %w", ociBin, networkName, err) } +// createV6OrDualNetwork creates a user-defined bridge network with IPv6 enabled, +// and adds both subnets when ipFamily == "dual". Returns the gateway reported by inspect (may be nil for v6-only). +func createV6OrDualNetwork(ociBin, name, subnetV4, subnetV6, ipFamily string) (net.IP, error) { + klog.Infof("creating %s network %q (family=%s, v4=%q, v6=%q)", ociBin, name, ipFamily, subnetV4, subnetV6) + + // If exists, reuse as-is + if info, err := containerNetworkInspect(ociBin, name); err == nil { + klog.Infof("found existing network %q", name) + return info.gateway, nil + } + + // Defaults + if subnetV6 == "" { + subnetV6 = firstSubnetAddrv6("") + } + + // Build args + args := []string{"network", "create", "--driver=bridge"} + if ipFamily == "ipv6" || ipFamily == "dual" { + args = append(args, "--ipv6") + } + if ipFamily == "dual" && subnetV4 != "" { + args = append(args, "--subnet", subnetV4) + } + if subnetV6 != "" { + args = append(args, "--subnet", subnetV6) + } + // Optional bridge knobs (safe to omit) + // if ociBin == Docker { + // args = append(args, "-o", "com.docker.network.bridge.enable_ip_masquerade=true") + // args = append(args, "-o", "com.docker.network.bridge.enable_icc=true") + // } + args = append(args, + fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), + fmt.Sprintf("--label=%s=%s", ProfileLabelKey, name), + name, + ) + + if _, err := runCmd(exec.Command(ociBin, args...)); err != nil { + klog.Warningf("failed to create %s network %q: %v", ociBin, name, err) + return nil, fmt.Errorf("create %s network %q: %w", ociBin, name, err) + } + + // Rely on inspect (gateway may be empty for IPv6) + info, err := containerNetworkInspect(ociBin, name) + if err != nil { + // non-fatal: just return nil gateway + klog.Warningf("post-create inspect failed for %s: %v", name, err) + return nil, nil + } + return info.gateway, nil +} + func tryCreateDockerNetwork(ociBin string, subnet *network.Parameters, mtu int, name string) (net.IP, error) { gateway := net.ParseIP(subnet.Gateway) klog.Infof("attempt to create %s network %s %s with gateway %s and MTU of %d ...", ociBin, name, subnet.CIDR, subnet.Gateway, mtu) @@ -185,22 +255,36 @@ func containerNetworkInspect(ociBin string, name string) (netInfo, error) { // networkInspect is only used to unmarshal the docker network inspect output and translate it to netInfo type networkInspect struct { - Name string - Driver string - Subnet string - Gateway string - MTU int - ContainerIPs []string + Name string `json:"Name"` + Driver string `json:"Driver"` + // Legacy single fields (older template) + Subnet string `json:"Subnet"` + Gateway string `json:"Gateway"` + // Multi-family (new template) + Subnets []string `json:"Subnets"` + Gateways []string `json:"Gateways"` + MTU int `json:"MTU"` + ContainerIPs []string `json:"ContainerIPs"` } + var dockerInspectGetter = func(name string) (*RunResult, error) { - // hack -- 'support ancient versions of docker again (template parsing issue) #10362' and resolve 'Template parsing error: template: :1: unexpected "=" in operand' / 'exit status 64' - // note: docker v18.09.7 and older use go v1.10.8 and older, whereas support for '=' operator in go templates came in go v1.11 - cmd := exec.Command(Docker, "network", "inspect", name, "--format", `{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}`) + // keep the old workaround: avoid eq/== in templates; emit trailing commas then strip ",]" + cmd := exec.Command( + Docker, "network", "inspect", name, "--format", + `{"Name":"{{.Name}}","Driver":"{{.Driver}}",` + + `"Subnet":"{{range .IPAM.Config}}{{.Subnet}}{{end}}",` + + `"Gateway":"{{range .IPAM.Config}}{{.Gateway}}{{end}}",` + + `"Subnets":[{{range .IPAM.Config}}{{if .Subnet}}"{{.Subnet}}",{{end}}{{end}}],` + + `"Gateways":[{{range .IPAM.Config}}{{if .Gateway}}"{{.Gateway}}",{{end}}{{end}}],` + + `"MTU":{{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}},` + + `"ContainerIPs":[{{range $k,$v := .Containers}}"{{$v.IPv4Address}}",{{end}}]}`, + ) rr, err := runCmd(cmd) - // remove extra ',' after the last element in the ContainerIPs slice - rr.Stdout = *bytes.NewBuffer(bytes.ReplaceAll(rr.Stdout.Bytes(), []byte(",]"), []byte("]"))) - return rr, err + // remove any trailing commas from arrays we just built + cleaned := bytes.ReplaceAll(rr.Stdout.Bytes(), []byte(",]"), []byte("]")) + rr.Stdout = *bytes.NewBuffer(cleaned) + return rr, err } // if exists returns subnet, gateway and mtu @@ -217,19 +301,51 @@ func dockerNetworkInspect(name string) (netInfo, error) { return info, err } - // results looks like {"Name": "bridge","Driver": "bridge","Subnet": "172.17.0.0/16","Gateway": "172.17.0.1","MTU": 1500, "ContainerIPs": ["172.17.0.3/16", "172.17.0.2/16"]} + // results look like: + // {"Name":"bridge","Driver":"bridge", + // "Subnet":"172.17.0.0/16","Gateway":"172.17.0.1", + // "Subnets":["172.17.0.0/16","fd00::/64"],"Gateways":["172.17.0.1","fd00::1"], + // "MTU":1500,"ContainerIPs":[...]} if err := json.Unmarshal(rr.Stdout.Bytes(), &vals); err != nil { return info, fmt.Errorf("error parsing network inspect output: %q", rr.Stdout.String()) } - info.gateway = net.ParseIP(vals.Gateway) - info.mtu = vals.MTU - - _, info.subnet, err = net.ParseCIDR(vals.Subnet) - if err != nil { - return info, errors.Wrapf(err, "parse subnet for %s", name) + // Choose a subnet/gateway: + // - Prefer an IPv4 entry (back-compat with existing IPv4 flows), + // - else fall back to first entry, + // - else use legacy single fields. + pickSubnet := "" + pickGateway := "" + if len(vals.Subnets) > 0 { + for i, s := range vals.Subnets { + if ip, _, e := net.ParseCIDR(s); e == nil && ip.To4() != nil { + pickSubnet = s + if i < len(vals.Gateways) { + pickGateway = vals.Gateways[i] + } + break + } + } + if pickSubnet == "" { + pickSubnet = vals.Subnets[0] + if len(vals.Gateways) > 0 { + pickGateway = vals.Gateways[0] + } + } } - + if pickSubnet == "" { + pickSubnet = vals.Subnet + pickGateway = vals.Gateway + } + if pickSubnet != "" { + if _, info.subnet, err = net.ParseCIDR(pickSubnet); err != nil { + return info, errors.Wrapf(err, "parse subnet for %s", name) + } + } + if pickGateway != "" { + info.gateway = net.ParseIP(pickGateway) + } + info.mtu = vals.MTU return info, nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 6370cb30ad48..445dcae58443 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -185,12 +185,38 @@ func CreateContainerNode(p CreateParams) error { //nolint to suppress cyclomatic // label th enode with the node ID "--label", p.NodeLabel, } - // to provide a static IP - if p.Network != "" && p.IP != "" { - runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) - } + // attach to the user-defined bridge network (once), and set static IPs if provided + if p.Network != "" { + runArgs = append(runArgs, "--network", p.Network) + if p.IP != "" { + runArgs = append(runArgs, "--ip", p.IP) // IPv4 + } + if p.IPv6 != "" { + runArgs = append(runArgs, "--ip6", p.IPv6) // IPv6 + } + } + + // For IPv6/dual clusters, enable forwarding inside the node container + // (safe sysctl; avoid disable_ipv6 which may be blocked by Docker's safe list) + + // Ensure service rules apply to bridged traffic inside the node container. + // Do both families; harmless if already set. + runArgs = append(runArgs, + "--sysctl", "net.ipv4.ip_forward=1", + "--sysctl", "net.bridge.bridge-nf-call-iptables=1", + ) + // IPv6/dual clusters need IPv6 forwarding and IPv6 bridge netfilter, too. + if p.IPFamily == "ipv6" || p.IPFamily == "dual" { + runArgs = append(runArgs, + "--sysctl", "net.ipv6.conf.all.forwarding=1", + "--sysctl", "net.bridge.bridge-nf-call-ip6tables=1", + // Allow kube-proxy/IPVS or iptables to program and accept Service VIPs. + "--sysctl", "net.ipv4.ip_nonlocal_bind=1", + // Same for IPv6 VIPs. + "--sysctl", "net.ipv6.ip_nonlocal_bind=1", + ) + } switch p.GPUs { case "all", "nvidia": runArgs = append(runArgs, "--gpus", "all", "--env", "NVIDIA_DRIVER_CAPABILITIES=all") @@ -509,7 +535,12 @@ func generatePortMappings(portMappings ...PortMapping) []string { for _, pm := range portMappings { // let docker pick a host port by leaving it as :: // example --publish=127.0.0.17::8443 will get a random host port for 8443 - publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort) + host := pm.ListenAddress + // If the listen address is an IPv6 literal, bracket it for Docker syntax. + if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") { + host = "[" + host + "]" + } + publish := fmt.Sprintf("--publish=%s::%d", host, pm.ContainerPort) result = append(result, publish) } return result diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 894679c4e667..4e58d84cfab8 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -61,6 +61,8 @@ type CreateParams struct { OCIBinary string // docker or podman Network string // network name that the container will attach to IP string // static IP to assign the container in the cluster network + IPv6 string // optional static IPv6 to assign to the node container (--ip6) + IPFamily string // "ipv4", "ipv6", or "dual" (from cc.KubernetesConfig.IPFamily) GPUs string // add GPU devices to the container } diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 780f7c0e45f2..f91f41cb0183 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -66,7 +66,10 @@ type Config struct { ContainerRuntime string // container runtime kic is running Network string // network to run with kic Subnet string // subnet to be used on kic cluster + Subnetv6 string StaticIP string // static IP for the kic cluster + StaticIPv6 string + IPFamily string // "ipv4", "ipv6", or "dual" ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... ListenAddress string // IP Address to listen to GPUs string // add GPU devices to the container diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go index 3e0d021f7245..ab7c8f299adf 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go @@ -93,7 +93,7 @@ staticPodPath: {{.StaticPodPath}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: {{.KubeProxyMetricsBindAddress}} conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go index fc098623cfed..2621e37e5ade 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go @@ -24,7 +24,7 @@ var V1Beta2 = template.Must(template.New("configTmpl-v1beta2").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -37,15 +37,25 @@ nodeRegistration: criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" kubeletExtraArgs: - node-ip: {{.NodeIP}} + node-ip: "{{.NodeIP}}" taints: [] --- apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} {{end}}{{range .ComponentOptions}}{{.Component}}: +{{- if eq .Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} {{- range $k, $v := .Pairs }} + {{- if not (and (eq .Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: {{- range $i, $val := printMapInOrder .ExtraArgs ": " }} @@ -72,7 +82,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -96,7 +106,7 @@ staticPodPath: {{.StaticPodPath}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go index 26de90f42e6f..c988d9c813ff 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go @@ -24,7 +24,7 @@ var V1Beta3 = template.Must(template.New("configTmpl-v1beta3").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -37,15 +37,25 @@ nodeRegistration: criSocket: {{if .CRISocket}}{{if .PrependCriSocketUnix}}unix://{{end}}{{.CRISocket}}{{else}}{{if .PrependCriSocketUnix}}unix://{{end}}/var/run/dockershim.sock{{end}} name: "{{.NodeName}}" kubeletExtraArgs: - node-ip: {{.NodeIP}} + node-ip: "{{.NodeIP}}" taints: [] --- apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} {{end}}{{range .ComponentOptions}}{{.Component}}: +{{- if eq .Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} {{- range $k, $v := .Pairs }} + {{- if not (and (eq .Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: {{- range $i, $val := printMapInOrder .ExtraArgs ": " }} @@ -70,7 +80,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -95,7 +105,7 @@ resolvConf: /etc/kubelet-resolv.conf{{end}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go index ee28b7d69da1..ca9055332bda 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta4.go @@ -24,7 +24,7 @@ var V1Beta4 = template.Must(template.New("configTmpl-v1beta4").Funcs(template.Fu }).Parse(`apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration localAPIEndpoint: - advertiseAddress: {{.AdvertiseAddress}} + advertiseAddress: "{{.AdvertiseAddress}}" bindPort: {{.APIServerPort}} bootstrapTokens: - groups: @@ -44,12 +44,24 @@ nodeRegistration: apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration {{ if .ImageRepository}}imageRepository: {{.ImageRepository}} -{{end}}{{range .ComponentOptions}}{{.Component}}: -{{- range $k, $v := .Pairs }} +{{end}}{{range .ComponentOptions }} + {{- $co := . }} +{{$co.Component}}: +{{- if eq $co.Component "apiServer" }} + {{- if $.APIServerCertSANs }} + certSANs: + {{- range $.APIServerCertSANs }} + - "{{ . }}" + {{- end }} + {{- end }} +{{- end }} +{{- range $k, $v := $co.Pairs }} + {{- if not (and (eq $co.Component "apiServer") (eq $k "certSANs")) }} {{$k}}: {{$v}} + {{- end }} {{- end}} extraArgs: -{{- range $key, $val := .ExtraArgs }} +{{- range $key, $val := $co.ExtraArgs }} - name: "{{$key}}" value: "{{$val}}" {{- end}} @@ -59,7 +71,7 @@ kind: ClusterConfiguration {{end -}}{{end -}} certificatesDir: {{.CertDir}} clusterName: mk -controlPlaneEndpoint: {{.ControlPlaneAddress}}:{{.APIServerPort}} +controlPlaneEndpoint: "{{.ControlPlaneEndpoint}}" etcd: local: dataDir: {{.EtcdDataDir}} @@ -73,7 +85,7 @@ kubernetesVersion: {{.KubernetesVersion}} networking: dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}} podSubnet: "{{.PodSubnet }}" - serviceSubnet: {{.ServiceCIDR}} + serviceSubnet: "{{.ServiceCIDR}}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -98,7 +110,7 @@ resolvConf: /etc/kubelet-resolv.conf{{end}} apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clusterCIDR: "{{.PodSubnet }}" -metricsBindAddress: 0.0.0.0:10249 +metricsBindAddress: "{{.KubeProxyMetricsBindAddress}}" conntrack: maxPerCore: 0 # Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established" diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index a65efb33d4f6..c477b382857e 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -21,6 +21,7 @@ import ( "bytes" "fmt" "path" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -76,13 +77,25 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana return nil, errors.Wrap(err, "cni") } - podCIDR := cnm.CIDR() - overrideCIDR := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm) - if overrideCIDR != "" { - podCIDR = overrideCIDR - } - klog.Infof("Using pod CIDR: %s", podCIDR) - + // Build podSubnet(s) based on IP family + family := strings.ToLower(k8s.IPFamily) + v4Pod := cnm.CIDR() + if o := k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm); o != "" { + v4Pod = o + } + var podSubnets []string + if family != "ipv6" && v4Pod != "" { + podSubnets = append(podSubnets, v4Pod) + } + if family != "ipv4" && k8s.PodCIDRv6 != "" { + podSubnets = append(podSubnets, k8s.PodCIDRv6) + } + podCIDR := strings.Join(podSubnets, ",") + if podCIDR != "" { + klog.Infof("Using pod subnet(s): %s", podCIDR) + } else { + klog.Infof("No pod subnet set via kubeadm (CNI will configure)") + } // ref: https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration kubeletConfigOpts := kubeletConfigOpts(k8s.ExtraOptions) // container-runtime-endpoint kubelet flag was deprecated but corresponding containerRuntimeEndpoint kubelet config field is "required" but supported only from k8s v1.27 @@ -107,11 +120,79 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana kubeletConfigOpts["runtimeRequestTimeout"] = "15m" } + // Build serviceSubnet(s) per IP family + // v4 default comes from constants; v6 must be provided via ServiceCIDRv6 + v4Svc := constants.DefaultServiceCIDR + if k8s.ServiceCIDR != "" { + v4Svc = k8s.ServiceCIDR + } + var svcSubnets []string + if family != "ipv6" && v4Svc != "" { + svcSubnets = append(svcSubnets, v4Svc) + } + if family != "ipv4" && k8s.ServiceCIDRv6 != "" { + svcSubnets = append(svcSubnets, k8s.ServiceCIDRv6) + } + serviceCIDR := strings.Join(svcSubnets, ",") + + // Choose advertise address & nodeIP according to family + advertiseAddress := n.IP + nodeIP := n.IP + if family == "ipv6" && n.IPv6 != "" { + advertiseAddress = n.IPv6 + nodeIP = n.IPv6 + } else if family == "dual" { + // let kubelet auto-detect both; don’t force a single family + nodeIP = "" + } + + cpEndpoint := fmt.Sprintf("%s:%d", constants.ControlPlaneAlias, nodePort) + if family == "ipv6" && advertiseAddress != "" { + cpEndpoint = fmt.Sprintf("[%s]:%d", advertiseAddress, nodePort) + } + +if family == "ipv6" { + ensured := false + for i := range componentOpts { + // match "apiServer" regardless of accidental casing + if strings.EqualFold(componentOpts[i].Component, "apiServer") { + if componentOpts[i].ExtraArgs == nil { + componentOpts[i].ExtraArgs = map[string]string{} + } + if _, ok := componentOpts[i].ExtraArgs["bind-address"]; !ok { + componentOpts[i].ExtraArgs["bind-address"] = "::" + } + // normalize the component name so the template emits 'apiServer' + componentOpts[i].Component = "apiServer" + ensured = true + break + } + } + if !ensured { + componentOpts = append(componentOpts, componentOptions{ + Component: "apiServer", + ExtraArgs: map[string]string{ + "bind-address": "::", + }, + }) + } +} + +apiServerCertSANs := []string{constants.ControlPlaneAlias} +switch strings.ToLower(k8s.IPFamily) { +case "ipv6": + apiServerCertSANs = append(apiServerCertSANs, "::1") +case "dual": + apiServerCertSANs = append(apiServerCertSANs, "127.0.0.1", "::1") +default: // ipv4 + apiServerCertSANs = append(apiServerCertSANs, "127.0.0.1") +} opts := struct { CertDir string ServiceCIDR string PodSubnet string AdvertiseAddress string + APIServerCertSANs []string APIServerPort int KubernetesVersion string EtcdDataDir string @@ -132,11 +213,14 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana ResolvConfSearchRegression bool KubeletConfigOpts map[string]string PrependCriSocketUnix bool + ControlPlaneEndpoint string + KubeProxyMetricsBindAddress string }{ CertDir: vmpath.GuestKubernetesCertsDir, - ServiceCIDR: constants.DefaultServiceCIDR, - PodSubnet: podCIDR, - AdvertiseAddress: n.IP, + ServiceCIDR: serviceCIDR, + PodSubnet: podCIDR, + AdvertiseAddress: advertiseAddress, + APIServerCertSANs: apiServerCertSANs, APIServerPort: nodePort, KubernetesVersion: k8s.KubernetesVersion, EtcdDataDir: EtcdDataDir(), @@ -149,7 +233,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana ComponentOptions: componentOpts, FeatureArgs: kubeadmFeatureArgs, DNSDomain: k8s.DNSDomain, - NodeIP: n.IP, + NodeIP: nodeIP, CgroupDriver: cgroupDriver, ClientCAFile: path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"), StaticPodPath: vmpath.GuestManifestsDir, @@ -157,10 +241,15 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana KubeProxyOptions: createKubeProxyOptions(k8s.ExtraOptions), ResolvConfSearchRegression: HasResolvConfSearchRegression(k8s.KubernetesVersion), KubeletConfigOpts: kubeletConfigOpts, - } - - if k8s.ServiceCIDR != "" { - opts.ServiceCIDR = k8s.ServiceCIDR + ControlPlaneEndpoint: cpEndpoint, + KubeProxyMetricsBindAddress: func() string { + switch strings.ToLower(k8s.IPFamily) { + case "ipv6": + return "[::]:10249" + default: // ipv4 or dual + return "0.0.0.0:10249" + } + }(), } configTmpl := ktmpl.V1Beta1 diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index e0811107eb42..a4f724ebdd4d 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path" + "strings" "github.com/blang/semver/v4" "github.com/pkg/errors" @@ -82,9 +83,24 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage } } - if _, ok := extraOpts["node-ip"]; !ok { - extraOpts["node-ip"] = nc.IP - } + // Pick node-ip based on requested IP family + if _, ok := extraOpts["node-ip"]; !ok { + family := strings.ToLower(k8s.IPFamily) + switch family { + case "ipv6": + if nc.IPv6 != "" { + extraOpts["node-ip"] = nc.IPv6 + } else { + // fallback if IPv6 wasn’t wired yet + extraOpts["node-ip"] = nc.IP + } + case "dual": + // Don’t set node-ip at all; kubelet will advertise both families. + // (If a user explicitly set node-ip, we honor it above.) + default: // "ipv4" or empty + extraOpts["node-ip"] = nc.IP + } + } if _, ok := extraOpts["hostname-override"]; !ok { nodeName := KubeNodeName(mc, nc) diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 3354f6bd8261..c518a577b7ef 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -256,14 +256,31 @@ func generateProfileCerts(cfg config.ClusterConfig, n config.Node, shared shared klog.Info("generating profile certs ...") k8s := cfg.KubernetesConfig - serviceIP, err := util.ServiceClusterIP(k8s.ServiceCIDR) - if err != nil { - return nil, errors.Wrap(err, "get service cluster ip") - } + if err != nil { + return nil, errors.Wrap(err, "get service cluster ip") + } + + // Collect both service VIPs if present + var serviceIPv6 net.IP + if k8s.ServiceCIDRv6 != "" { + if sip6, err := util.ServiceClusterIP(k8s.ServiceCIDRv6); err == nil { + serviceIPv6 = sip6 + } else { + klog.Warningf("failed to compute service IPv6 from %q: %v", k8s.ServiceCIDRv6, err) + } + } + + apiServerIPs := append([]net.IP{}, k8s.APIServerIPs...) + apiServerIPs = append(apiServerIPs, serviceIP) + if serviceIPv6 != nil { + apiServerIPs = append(apiServerIPs, serviceIPv6) + } + // Always include loopbacks for both families; the docker driver publishes ports on ::1. + apiServerIPs = append(apiServerIPs, net.ParseIP("127.0.0.1"), net.ParseIP("::1")) + // Common local addresses used by the node runtime/bridge + apiServerIPs = append(apiServerIPs, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")) - apiServerIPs := append([]net.IP{}, k8s.APIServerIPs...) - apiServerIPs = append(apiServerIPs, serviceIP, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")) // append ip addresses of all control-plane nodes for _, n := range config.ControlPlanes(cfg) { apiServerIPs = append(apiServerIPs, net.ParseIP(n.IP)) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 65500a332b94..260b4a453b9c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -386,6 +386,45 @@ func (k *Bootstrapper) unpause(cfg config.ClusterConfig) error { return nil } + +// ensureControlPlaneAlias adds control-plane.minikube.internal -> IP mapping in /etc/hosts +func (k *Bootstrapper) ensureControlPlaneAlias(cfg config.ClusterConfig) error { + family := strings.ToLower(cfg.KubernetesConfig.IPFamily) + + // HA: use the VIP that kube-vip sets + if config.IsHA(cfg) { + if ip := net.ParseIP(cfg.KubernetesConfig.APIServerHAVIP); ip != nil { + return machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip) + } + return nil + } + + // Single-CP: pick the right address based on IP family + cp, err := config.ControlPlane(cfg) + if err != nil { + return errors.Wrap(err, "get control-plane node") + } + + // For ipv6-only or dual, add AAAA + if family == "ipv6" || family == "dual" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv6)") + } + } + } + // For ipv4-only or dual, add A + if family != "ipv6" { + if ip4 := net.ParseIP(cp.IP); ip4 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip4); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv4)") + } + } + } + return nil +} + + // StartCluster starts the cluster func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error { start := time.Now() @@ -423,6 +462,10 @@ func (k *Bootstrapper) StartCluster(cfg config.ClusterConfig) error { return errors.Wrap(err, "cp") } + if err := k.ensureControlPlaneAlias(cfg); err != nil { + klog.Warningf("could not ensure control-plane alias: %v", err) + } + err := k.init(cfg) if err == nil { return nil @@ -743,6 +786,18 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return nil } + +func advertiseIP(cc config.ClusterConfig, n config.Node) string { + switch strings.ToLower(cc.KubernetesConfig.IPFamily) { + case "ipv6": + if n.IPv6 != "" { + return n.IPv6 + } + } + // default / ipv4 / dual: keep IPv4 + return n.IP +} + // JoinCluster adds new node to an existing cluster. func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinCmd string) error { // Join the control plane by specifying its token @@ -755,8 +810,10 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC // ref: https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-join/#options // "If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on. If not set the default network interface will be used." // "If the node should host a new control plane instance, the port for the API Server to bind to." - joinCmd += " --apiserver-advertise-address=" + n.IP + - " --apiserver-bind-port=" + strconv.Itoa(n.Port) + // pick IPv6 for ipv6 clusters, otherwise IPv4 + addr := advertiseIP(cc, n) + joinCmd += " --apiserver-advertise-address=" + addr + + " --apiserver-bind-port=" + strconv.Itoa(n.Port) } if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", joinCmd)); err != nil { @@ -996,18 +1053,45 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru // add "control-plane.minikube.internal" dns alias // note: needs to be called after APIServerHAVIP is set (in startPrimaryControlPlane()) and before kubeadm kicks off - cpIP := cfg.KubernetesConfig.APIServerHAVIP - if !config.IsHA(cfg) { - cp, err := config.ControlPlane(cfg) - if err != nil { - return errors.Wrap(err, "get control-plane node") - } - cpIP = cp.IP - } - if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, net.ParseIP(cpIP)); err != nil { - return errors.Wrap(err, "add control-plane alias") - } - + + family := strings.ToLower(cfg.KubernetesConfig.IPFamily) + +if config.IsHA(cfg) { + // For HA we already have APIServerHAVIP set appropriately by kube-vip generation + if ip := net.ParseIP(cfg.KubernetesConfig.APIServerHAVIP); ip != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip); err != nil { + return errors.Wrap(err, "add HA control-plane alias") + } + } +} else { + cp, err := config.ControlPlane(cfg) + if err != nil { + return errors.Wrap(err, "get control-plane node") + } + + // ipv6-only → write AAAA; ipv4/dual → write A; dual → write both + if family == "ipv6" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv6)") + } + } + } else { + if ip4 := net.ParseIP(cp.IP); ip4 != nil { + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip4); err != nil { + return errors.Wrap(err, "add control-plane alias (ipv4)") + } + } + if family == "dual" { + if ip6 := net.ParseIP(cp.IPv6); ip6 != nil { + // add AAAA alongside A + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, ip6); err != nil { + return errors.Wrap(err, "add control-plane alias (dual, ipv6)") + } + } + } + } +} // "ensure" kubelet is started, intentionally non-fatal in case of an error if err := sysinit.New(k.c).Start("kubelet"); err != nil { klog.Errorf("Couldn't ensure kubelet is started this might cause issues (will continue): %v", err) diff --git a/pkg/minikube/cni/bridge.go b/pkg/minikube/cni/bridge.go index d4264920ddb0..6e6b6e0f9348 100644 --- a/pkg/minikube/cni/bridge.go +++ b/pkg/minikube/cni/bridge.go @@ -17,10 +17,10 @@ limitations under the License. package cni import ( - "bytes" + "encoding/json" "fmt" "os/exec" - "text/template" + "strings" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" @@ -32,38 +32,72 @@ import ( // ref: https://www.cni.dev/plugins/current/meta/portmap/ // ref: https://www.cni.dev/plugins/current/meta/firewall/ -// note: "cannot set hairpin mode and promiscuous mode at the same time" -// ref: https://github.com/containernetworking/plugins/blob/7e9ada51e751740541969e1ea5a803cbf45adcf3/plugins/main/bridge/bridge.go#L424 -var bridgeConf = template.Must(template.New("bridge").Parse(` -{ - "cniVersion": "0.4.0", - "name": "bridge", - "plugins": [ - { - "type": "bridge", - "bridge": "bridge", - "addIf": "true", - "isDefaultGateway": true, - "forceAddress": false, - "ipMasq": true, - "hairpinMode": true, - "ipam": { - "type": "host-local", - "subnet": "{{.PodCIDR}}" - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - }, - { - "type": "firewall" - } - ] + +// renderBridgeConflist builds a bridge CNI config that supports IPv4-only, IPv6-only, or dual-stack. +func renderBridgeConflist(k8s config.KubernetesConfig) ([]byte, error) { + // minimal structs for JSON marshal + type rng struct{ Subnet string `json:"subnet"` } + type ipam struct { + Type string `json:"type"` + Subnet string `json:"subnet,omitempty"` // single-stack (v4 or v6) + Ranges [][]rng `json:"ranges,omitempty"` // dual-stack + } + type bridge struct { + Type string `json:"type"` + Bridge string `json:"bridge"` + IsDefaultGateway bool `json:"isDefaultGateway"` + HairpinMode bool `json:"hairpinMode"` + IPMasq bool `json:"ipMasq"` + IPAM ipam `json:"ipam"` + } + type plugin struct { + Type string `json:"type"` + Capabilities map[string]bool `json:"capabilities,omitempty"` + } + type conflist struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Plugins []interface{} `json:"plugins"` + } + + v4 := k8s.PodCIDR != "" + v6 := k8s.PodCIDRv6 != "" + + cfgIPAM := ipam{Type: "host-local"} + switch { + case v4 && v6: + cfgIPAM.Ranges = [][]rng{{{Subnet: k8s.PodCIDR}}, {{Subnet: k8s.PodCIDRv6}}} + case v6: + cfgIPAM.Subnet = k8s.PodCIDRv6 + default: + // fall back to previous default if unset upstream + cidr := k8s.PodCIDR + if cidr == "" { + cidr = DefaultPodCIDR + } + cfgIPAM.Subnet = cidr + } + + // NAT generally not desired for IPv6; keep masquerade only for v4 + ipMasq := v4 && !v6 + + br := bridge{ + Type: "bridge", Bridge: "cni0", + IsDefaultGateway: true, + HairpinMode: true, + IPMasq: ipMasq, + IPAM: cfgIPAM, + } + portmap := plugin{Type: "portmap", Capabilities: map[string]bool{"portMappings": true}} + firewall := plugin{Type: "firewall"} + + out := conflist{ + CNIVersion: "1.0.0", + Name: "k8s-pod-network", + Plugins: []interface{}{br, portmap, firewall}, + } + return json.MarshalIndent(out, "", " ") } -`)) // Bridge is a simple CNI manager for single-node usage type Bridge struct { @@ -76,14 +110,11 @@ func (c Bridge) String() string { } func (c Bridge) netconf() (assets.CopyableFile, error) { - input := &tmplInput{PodCIDR: DefaultPodCIDR} - - b := bytes.Buffer{} - if err := bridgeConf.Execute(&b, input); err != nil { - return nil, err - } - - return assets.NewMemoryAssetTarget(b.Bytes(), "/etc/cni/net.d/1-k8s.conflist", "0644"), nil + cfgBytes, err := renderBridgeConflist(c.cc.KubernetesConfig) + if err != nil { + return nil, err + } + return assets.NewMemoryAssetTarget(cfgBytes, "/etc/cni/net.d/1-k8s.conflist", "0644"), nil } // Apply enables the CNI @@ -110,5 +141,15 @@ func (c Bridge) Apply(r Runner) error { // CIDR returns the default CIDR used by this CNI func (c Bridge) CIDR() string { - return DefaultPodCIDR + + // Prefer explicitly-set CIDRs from the cluster config. + k := c.cc.KubernetesConfig + if k.PodCIDRv6 != "" && (strings.ToLower(k.IPFamily) == "ipv6" || k.PodCIDR == "") { + return k.PodCIDRv6 + } + + if k.PodCIDR != "" { + return k.PodCIDR + } + return DefaultPodCIDR } diff --git a/pkg/minikube/cni/calico.go b/pkg/minikube/cni/calico.go index 6f7b55c4b7c1..0d48741b3bfd 100644 --- a/pkg/minikube/cni/calico.go +++ b/pkg/minikube/cni/calico.go @@ -19,6 +19,9 @@ package cni import ( "bytes" "fmt" + "strings" + "time" + "os/exec" // goembed needs this _ "embed" @@ -29,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/util" ) @@ -46,7 +50,14 @@ type Calico struct { } type calicoTmplStruct struct { + // IPv4/IPv6/dual inputs for the template + IPFamily string PodCIDR string + PodCIDRv6 string + ServiceCIDR string + ServiceCIDRv6 string + ControlPlaneAlias string + APIServerPort int DeploymentImageName string DaemonSetImageName string BinaryImageName string @@ -64,12 +75,39 @@ func (c Calico) manifest() (assets.CopyableFile, error) { if err != nil { return nil, fmt.Errorf("failed to parse Kubernetes version: %v", err) } + + k := c.cc.KubernetesConfig + fam := strings.ToLower(k.IPFamily) + // Defaults/fallbacks to stay safe if flags weren’t provided + v4Pod := k.PodCIDR + if v4Pod == "" { + v4Pod = DefaultPodCIDR + } + v6Pod := k.PodCIDRv6 + svcV4 := k.ServiceCIDR + if svcV4 == "" { + svcV4 = constants.DefaultServiceCIDR + } + svcV6 := k.ServiceCIDRv6 + if svcV6 == "" && fam != "ipv4" { + svcV6 = constants.DefaultServiceCIDRv6 + } + apiPort := c.cc.APIServerPort + if apiPort == 0 { + apiPort = 8443 + } input := &calicoTmplStruct{ - PodCIDR: DefaultPodCIDR, - DeploymentImageName: images.CalicoDeployment(c.cc.KubernetesConfig.ImageRepository), - DaemonSetImageName: images.CalicoDaemonSet(c.cc.KubernetesConfig.ImageRepository), - BinaryImageName: images.CalicoBin(c.cc.KubernetesConfig.ImageRepository), + IPFamily: fam, + PodCIDR: v4Pod, + PodCIDRv6: v6Pod, + ServiceCIDR: svcV4, + ServiceCIDRv6: svcV6, + ControlPlaneAlias: constants.ControlPlaneAlias, + APIServerPort: apiPort, + DeploymentImageName: images.CalicoDeployment(k.ImageRepository), + DaemonSetImageName: images.CalicoDaemonSet(k.ImageRepository), + BinaryImageName: images.CalicoBin(k.ImageRepository), LegacyPodDisruptionBudget: k8sVersion.LT(semver.Version{Major: 1, Minor: 25}), } @@ -86,11 +124,85 @@ func (c Calico) Apply(r Runner) error { if err != nil { return errors.Wrap(err, "manifest") } - return applyManifest(c.cc, r, m) + // Phase 1: apply core Calico (includes CRDs) + if err := applyManifest(c.cc, r, m); err != nil { + return err + } + + // Phase 2: wait for CRD to be Established, then apply IPPools + if err := waitForCRDEstablished(r, c.cc.KubernetesConfig.KubernetesVersion, "ippools.crd.projectcalico.org", 90*time.Second); err != nil { + // Non-fatal: log and try to continue; but usually this must succeed. + return errors.Wrap(err, "waiting for Calico IPPool CRD") + } + + ipPoolsYAML := renderCalicoIPPools(c.cc.KubernetesConfig) + if ipPoolsYAML == "" { + return nil + } + poolsAsset := assets.NewMemoryAssetTarget([]byte(ipPoolsYAML), "/var/tmp/minikube/calico-ippools.yaml", "0644") + return applyManifest(c.cc, r, poolsAsset) +} + +// waitForCRDEstablished waits until the given CRD reports Established=True. +func waitForCRDEstablished(r Runner, k8sVersion string, crd string, to time.Duration) error { + kubectlPath := fmt.Sprintf("/var/lib/minikube/binaries/%s/kubectl", k8sVersion) + cmd := exec.Command("sudo", kubectlPath, "wait", + "--kubeconfig=/var/lib/minikube/kubeconfig", + "--for=condition=Established", + fmt.Sprintf("--timeout=%ds", int(to.Seconds())), + "crd/"+crd, + ) + _, err := r.RunCmd(cmd) + return err +} + +// renderCalicoIPPools returns a small manifest for IPv4/IPv6 IPPools based on IPFamily and PodCIDRs. +func renderCalicoIPPools(k config.KubernetesConfig) string { +fam := strings.ToLower(k.IPFamily) + var b strings.Builder + emit := func(name, cidr string, nat bool) { + if cidr == "" { + return + } + if b.Len() > 0 { + b.WriteString("\n---\n") + } + fmt.Fprintf(&b, "apiVersion: crd.projectcalico.org/v1\nkind: IPPool\nmetadata:\n name: %s\nspec:\n cidr: %q\n natOutgoing: %t\n nodeSelector: \"all()\"\n", name, cidr, nat) + } +// IPv4 pool unless explicitly ipv6-only + if fam != "ipv6" { + cidr := k.PodCIDR + if cidr == "" { + cidr = DefaultPodCIDR + } + emit("default-ipv4-ippool", cidr, true) + } + // IPv6 pool for ipv6 or dual + if fam == "ipv6" || fam == "dual" { + cidr := k.PodCIDRv6 + if cidr == "" { + // default provided by your constants if you prefer: + // cidr = constants.DefaultPodCIDRv6 + // but safer to require user/normalizer to have set it. + cidr = constants.DefaultPodCIDRv6 + } + emit("default-ipv6-ippool", cidr, false) + } + return b.String() } // CIDR returns the default CIDR used by this CNI func (c Calico) CIDR() string { // Calico docs specify 192.168.0.0/16 - but we do this for compatibility with other CNI's. + k := c.cc.KubernetesConfig + fam := strings.ToLower(k.IPFamily) + // Prefer explicitly-set CIDRs; for ipv6-only prefer v6 + if k.PodCIDRv6 != "" && (fam == "ipv6" || k.PodCIDR == "") { + return k.PodCIDRv6 + } + if k.PodCIDR != "" { + return k.PodCIDR + } + // fallback for legacy behavior return DefaultPodCIDR } diff --git a/pkg/minikube/cni/calico.yaml b/pkg/minikube/cni/calico.yaml index cd28d1f464c4..f8104a33db65 100644 --- a/pkg/minikube/cni/calico.yaml +++ b/pkg/minikube/cni/calico.yaml @@ -9881,6 +9881,21 @@ spec: name: kubernetes-services-endpoint optional: true env: + + - name: FELIX_IPV6SUPPORT + value: '{{ if or (eq .IPFamily "ipv6") (eq .IPFamily "dual") }}true{{ else }}false{{ end }}' + - name: IP + value: "autodetect" + - name: IP6 + value: '{{ if or (eq .IPFamily "ipv6") (eq .IPFamily "dual") }}autodetect{{ else }}none{{ end }}' + - name: CALICO_IPV4POOL_CIDR + value: "{{ .PodCIDR }}" + - name: CALICO_IPV6POOL_CIDR + value: "{{ .PodCIDRv6 }}" + - name: CALICO_IPV4POOL_NAT_OUTGOING + value: "Enabled" + - name: CALICO_IPV6POOL_NAT_OUTGOING + value: "Disabled" - name: KUBERNETES_NODE_NAME valueFrom: fieldRef: @@ -10175,6 +10190,10 @@ spec: labels: k8s-app: calico-kube-controllers spec: + {{- if eq .IPFamily "ipv6" }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- end }} nodeSelector: kubernetes.io/os: linux tolerations: @@ -10197,9 +10216,17 @@ spec: env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node,loadbalancer + value: '{{ if eq .IPFamily "ipv6" }}node{{ else }}node,loadbalancer{{ end }}' - name: DATASTORE_TYPE value: kubernetes + {{- if eq .IPFamily "ipv6" }} + # Avoid DNS and Service VIPs during IPv6 bootstrap: + # kube-controllers runs with hostNetwork, so loopback reaches the local apiserver. + - name: KUBERNETES_SERVICE_HOST + value: "::1" + - name: KUBERNETES_SERVICE_PORT + value: "{{ .APIServerPort }}" + {{- end }} livenessProbe: exec: command: diff --git a/pkg/minikube/cni/cilium.go b/pkg/minikube/cni/cilium.go index f02728f6b902..fb4fd3519810 100644 --- a/pkg/minikube/cni/cilium.go +++ b/pkg/minikube/cni/cilium.go @@ -23,6 +23,7 @@ import ( "io" "os/exec" "text/template" + "strings" "github.com/blang/semver/v4" "github.com/icza/dyno" @@ -32,6 +33,9 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/util" ) +// Default IPv6 pod CIDR when running Cilium with IPv6 enabled +// (Each node will get a /64 slice from this /48 by default.) +const DefaultPodCIDRv6 = "fd01::/48" // Generated by running `make update-cilium-version` // @@ -64,15 +68,68 @@ func (c Cilium) GenerateCiliumYAML() ([]byte, error) { } } - podCIDR := DefaultPodCIDR + // Decide IPv4/IPv6 family. Prefer explicit v6 fields if provided, otherwise + // fall back to parsing a comma-separated ServiceCIDR string. + svcCIDR := c.cc.KubernetesConfig.ServiceCIDR + svcCIDRv6 := c.cc.KubernetesConfig.ServiceCIDRv6 + partHasV6 := func(s string) bool { return strings.Contains(s, ":") } + partHasV4 := func(s string) bool { return strings.Contains(s, ".") } + enableV6 := false + enableV4 := true + if svcCIDR != "" { + if strings.Contains(svcCIDR, ",") { + for _, p := range strings.Split(svcCIDR, ",") { + p = strings.TrimSpace(p) + if partHasV6(p) { enableV6 = true } + if partHasV4(p) { enableV4 = true } + } + } else { + enableV6 = partHasV6(svcCIDR) + if enableV6 && !partHasV4(svcCIDR) { + enableV4 = false + } + } + } + + // If the cluster was configured with a dedicated IPv6 service range, honor it. + if svcCIDRv6 != "" && partHasV6(svcCIDRv6) { + enableV6 = true + // If no IPv4 service range was provided at all, assume IPv6-only + if svcCIDR == "" { + enableV4 = false + } + } + + podCIDRv4 := DefaultPodCIDR + podCIDRv6 := DefaultPodCIDRv6 + // If a specific v6 pod subnet was provided (e.g. via --subnet-v6), prefer it. + if v := c.cc.Subnetv6; v != "" { + podCIDRv6 = v + } + // Valid values in Cilium v1.18: disabled | partial | strict + // Safe default (kube-proxy addon is typically enabled in minikube): partial + kprMode := "partial" + // If you later decide to disable the kube-proxy addon, flip this to "strict". + // if enableV6 { kprMode = "strict" } - klog.Infof("Using pod CIDR: %s", podCIDR) + klog.Infof("Cilium IP family: enableIPv4=%t enableIPv6=%t", enableV4, enableV6) + klog.Infof("Using pod CIDRs: v4=%s v6=%s", podCIDRv4, podCIDRv6) + klog.Infof("Cilium kube-proxy-replacement: %s", kprMode) opts := struct { - PodSubnet string - }{ - PodSubnet: podCIDR, - } + PodSubnet string + PodSubnetV6 string + EnableIPv4 string + EnableIPv6 string + KubeProxyReplacement string + }{ + PodSubnet: podCIDRv4, + PodSubnetV6: podCIDRv6, + EnableIPv4: fmt.Sprintf("%t", enableV4), + EnableIPv6: fmt.Sprintf("%t", enableV6), + KubeProxyReplacement: kprMode, + } + ciliumTmpl := template.Must(template.New("name").Parse(ciliumYaml)) b := bytes.Buffer{} configTmpl := ciliumTmpl @@ -114,7 +171,7 @@ func removeAppArmorProfile(ciliumConfig string) (string, error) { } else if err != nil { return "", fmt.Errorf("failed to unmarshal yaml: %v", err) } - if err := dyno.Delete(obj, "appArmorProfile", "spec", "template", "spec", "securityContext"); err != nil { + if err := dyno.Delete(obj, "spec", "template", "spec", "securityContext", "appArmorProfile"); err != nil { return "", fmt.Errorf("failed to remove securityContext yaml: %v", err) } if err := encoder.Encode(obj); err != nil { diff --git a/pkg/minikube/cni/cilium.yaml b/pkg/minikube/cni/cilium.yaml index f14b106c9b43..9bd90b1dc7b1 100644 --- a/pkg/minikube/cni/cilium.yaml +++ b/pkg/minikube/cni/cilium.yaml @@ -79,11 +79,11 @@ data: # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. - enable-ipv4: "true" + enable-ipv4: "{{ .EnableIPv4 }}" # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 # address. - enable-ipv6: "false" + enable-ipv6: "{{ .EnableIPv6 }}" # Users who wish to specify their own custom CNI configuration file must set # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. custom-cni-conf: "false" @@ -163,10 +163,10 @@ data: # Enables L7 proxy for L7 policy enforcement and visibility enable-l7-proxy: "true" - enable-ipv4-masquerade: "true" + enable-ipv4-masquerade: "{{ if eq .EnableIPv4 "true" }}true{{ else }}false{{ end }}" enable-ipv4-big-tcp: "false" enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "true" + enable-ipv6-masquerade: "{{ if eq .EnableIPv6 "true" }}true{{ else }}false{{ end }}" enable-tcx: "true" datapath-mode: "veth" enable-masquerade-to-route-source: "false" @@ -219,8 +219,14 @@ data: hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt ipam: "cluster-pool" ipam-cilium-node-update-rate: "15s" + {{- if eq .EnableIPv4 "true" }} cluster-pool-ipv4-cidr: "{{ .PodSubnet }}" cluster-pool-ipv4-mask-size: "24" + {{- end }} + {{- if eq .EnableIPv6 "true" }} + cluster-pool-ipv6-cidr: "{{ .PodSubnetV6 }}" + cluster-pool-ipv6-mask-size: "64" + {{- end }} default-lb-service-ipam: "lbipam" egress-gateway-reconciliation-trigger-interval: "1s" diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 1077ac3ca255..a866ed4d1a18 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -50,6 +50,7 @@ type ClusterConfig struct { InsecureRegistry []string RegistryMirror []string HostOnlyCIDR string // Only used by the virtualbox driver + HostOnlyCIDRv6 string // IPv6 CIDR for the virtualbox driver HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string @@ -85,6 +86,7 @@ type ClusterConfig struct { ListenAddress string // Only used by the docker and podman driver Network string // only used by docker driver Subnet string // only used by the docker and podman driver + Subnetv6 string // IPv6 subnet for docker and podman driver MultiNodeRequested bool ExtraDisks int // currently only implemented for hyperkit and kvm2 CertExpiration time.Duration @@ -105,6 +107,7 @@ type ClusterConfig struct { SocketVMnetClientPath string SocketVMnetPath string StaticIP string + StaticIPv6 string // Static IPv6 address for the cluster SSHAuthSock string SSHAgentPID int GPUs string @@ -126,6 +129,10 @@ type KubernetesConfig struct { NetworkPlugin string FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ ServiceCIDR string // the subnet which Kubernetes services will be deployed to + ServiceCIDRv6 string // the IPv6 subnet which Kubernetes services will be deployed to + PodCIDR string // the IPv4 subnet which Kubernetes pods will be deployed to + PodCIDRv6 string // the IPv6 subnet which Kubernetes pods will be deployed to + IPFamily string // IP family mode: ipv4, ipv6, or dual ImageRepository string LoadBalancerStartIP string // currently only used by MetalLB addon LoadBalancerEndIP string // currently only used by MetalLB addon @@ -143,6 +150,7 @@ type KubernetesConfig struct { type Node struct { Name string IP string + IPv6 string // IPv6 address of the node Port int KubernetesVersion string ContainerRuntime string diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 4e8ef95a1bab..3df12506bf08 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -80,6 +80,10 @@ const ( ClusterDNSDomain = "cluster.local" // DefaultServiceCIDR is The CIDR to be used for service cluster IPs DefaultServiceCIDR = "10.96.0.0/12" + // DefaultServiceCIDRv6 is The IPv6 CIDR to be used for service cluster IPs + DefaultServiceCIDRv6 = "fd00::/108" + // DefaultPodCIDRv6 is The IPv6 CIDR to be used for pod IPs + DefaultPodCIDRv6 = "fd01::/64" // HostAlias is a DNS alias to the container/VM host IP HostAlias = "host.minikube.internal" // ControlPlaneAlias is a DNS alias pointing to the apiserver frontend diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 85699fa0d375..5d5d68dd5150 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -159,21 +159,26 @@ func (r *CRIO) Active() bool { // enableIPForwarding configures IP forwarding, which is handled normally by Docker // Context: https://github.com/kubernetes/kubeadm/issues/1062 func enableIPForwarding(cr CommandRunner) error { - // The bridge-netfilter module enables iptables rules to work on Linux bridges - // NOTE: br_netfilter isn't available in WSL2, but forwarding works fine there anyways - c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables") - if rr, err := cr.RunCmd(c); err != nil { - klog.Infof("couldn't verify netfilter by %q which might be okay. error: %v", rr.Command(), err) - c = exec.Command("sudo", "modprobe", "br_netfilter") - if _, err := cr.RunCmd(c); err != nil { - klog.Warningf("%q failed, which may be ok: %v", rr.Command(), err) - } - } - c = exec.Command("sudo", "sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrapf(err, "ip_forward") - } - return nil + // The bridge-netfilter module enables (ip|ip6)tables rules to apply on Linux bridges. + // NOTE: br_netfilter isn't available everywhere (e.g., some WSL2 kernels) – treat as best-effort. + if _, err := cr.RunCmd(exec.Command("sudo", "modprobe", "br_netfilter")); err != nil { + klog.Warningf("modprobe br_netfilter failed (may be OK on this kernel): %v", err) + } + + // Enable bridge netfilter hooks for both IPv4 and IPv6, and enable forwarding. + // Best-effort: warn but don't fail hard if a sysctl isn't present. + sysctls := []string{ + "sysctl -w net.bridge.bridge-nf-call-iptables=1", + "sysctl -w net.bridge.bridge-nf-call-ip6tables=1", + "sysctl -w net.ipv4.ip_forward=1", + "sysctl -w net.ipv6.conf.all.forwarding=1", + } + for _, s := range sysctls { + if _, err := cr.RunCmd(exec.Command("sudo", "sh", "-c", s)); err != nil { + klog.Warningf("failed to run %q (continuing): %v", s, err) + } + } + return nil } // enableRootless enables configurations for running CRI-O in Rootless Docker. diff --git a/pkg/minikube/driver/endpoint.go b/pkg/minikube/driver/endpoint.go index d5e86a746e57..73fa793634cc 100644 --- a/pkg/minikube/driver/endpoint.go +++ b/pkg/minikube/driver/endpoint.go @@ -19,6 +19,7 @@ package driver import ( "fmt" "net" + "strings" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" @@ -27,7 +28,6 @@ import ( "k8s.io/minikube/pkg/network" ) -// ControlPlaneEndpoint returns the location where callers can reach this cluster. func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) { if NeedsPortForward(driverName) { port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port) @@ -35,35 +35,58 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName klog.Warningf("failed to get forwarded control plane port %v", err) } - hostname := oci.DaemonHost(driverName) - ips, err := net.LookupIP(hostname) - if err != nil || len(ips) == 0 { - return hostname, nil, port, fmt.Errorf("failed to lookup ip for %q", hostname) - } + // Start with daemon host (docker/podman), tweak for IPv6, then honor APIServerName override. + host := oci.DaemonHost(driverName) + // If the cluster/node IP is IPv6 and daemon host is localhost on IPv4, + // force IPv6 loopback so we hit the port that’s actually listening. + if strings.Contains(cp.IP, ":") && (host == "127.0.0.1" || host == "localhost") { + host = "::1" + } + if cc.KubernetesConfig.APIServerName != constants.APIServerName { + host = cc.KubernetesConfig.APIServerName + } - // https://github.com/kubernetes/minikube/issues/3878 - if cc.KubernetesConfig.APIServerName != constants.APIServerName { - hostname = cc.KubernetesConfig.APIServerName + // Resolve final host -> IPs. Allow literal IPv4/IPv6 without DNS. + var ips []net.IP + if ip := net.ParseIP(host); ip != nil { + ips = []net.IP{ip} + } else { + ips, err = net.LookupIP(host) + if err != nil || len(ips) == 0 { + return host, nil, port, fmt.Errorf("failed to lookup ip for %q", host) + } } - return hostname, ips[0], port, nil + + return host, ips[0], port, nil } if IsQEMU(driverName) && network.IsBuiltinQEMU(cc.Network) { - return "localhost", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil + if strings.Contains(cp.IP, ":") { + return "::1", net.IPv6loopback, cc.APIServerPort, nil + } + return "127.0.0.1", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil } - // https://github.com/kubernetes/minikube/issues/3878 - hostname := cp.IP + // Default: use the node IP (literal or resolvable name) + host := cp.IP if cc.KubernetesConfig.APIServerName != constants.APIServerName { - hostname = cc.KubernetesConfig.APIServerName + host = cc.KubernetesConfig.APIServerName } - ips, err := net.LookupIP(cp.IP) - if err != nil || len(ips) == 0 { - return hostname, nil, cp.Port, fmt.Errorf("failed to lookup ip for %q", cp.IP) + + var ips []net.IP + if ip := net.ParseIP(cp.IP); ip != nil { + ips = []net.IP{ip} + } else { + var err error + ips, err = net.LookupIP(cp.IP) + if err != nil || len(ips) == 0 { + return host, nil, cp.Port, fmt.Errorf("failed to lookup ip for %q", cp.IP) + } } - return hostname, ips[0], cp.Port, nil + + return host, ips[0], cp.Port, nil } // AutoPauseProxyEndpoint returns the endpoint for the auto-pause (reverse proxy to api-sever) diff --git a/pkg/minikube/kubeconfig/kubeconfig.go b/pkg/minikube/kubeconfig/kubeconfig.go index 29a9419f0f6b..4e83b4945def 100644 --- a/pkg/minikube/kubeconfig/kubeconfig.go +++ b/pkg/minikube/kubeconfig/kubeconfig.go @@ -23,6 +23,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" @@ -52,7 +53,13 @@ func UpdateEndpoint(contextName string, host string, port int, configPath string return false, errors.Wrap(err, "get kubeconfig") } - address := "https://" + host + ":" + strconv.Itoa(port) + // Bracket IPv6 literals for a valid URL (e.g. https://[::1]:8443) + hostForURL := host + if strings.Contains(hostForURL, ":") && !strings.HasPrefix(hostForURL, "[") { + hostForURL = "[" + hostForURL + "]" + } + + address := "https://" + hostForURL + ":" + strconv.Itoa(port) // check & fix kubeconfig if the cluster or context setting is missing, or server address needs updating errs := configIssues(cfg, contextName, address) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 12ca77f21c9c..0fb0cd3e998b 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -24,6 +24,7 @@ import ( "os/exec" "path" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -399,31 +400,54 @@ func showHostInfo(h *host.Host, cfg config.ClusterConfig) { out.Step(style.StartingVM, "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize, "machine_type": machineType}) } -// AddHostAlias makes fine adjustments to pod resources that aren't possible via kubeadm config. +// AddHostAlias ensures /etc/hosts contains an entry for name -> ip. +// It preserves an existing record of the other IP family (so dual-stack can have both A and AAAA). func AddHostAlias(c command.Runner, name string, ip net.IP) error { - record := fmt.Sprintf("%s\t%s", ip, name) - if _, err := c.RunCmd(exec.Command("grep", record+"$", "/etc/hosts")); err == nil { - return nil - } - - if _, err := c.RunCmd(addHostAliasCommand(name, record, true, "/etc/hosts")); err != nil { - return errors.Wrap(err, "hosts update") - } - return nil + if ip == nil || ip.IsUnspecified() || ip.String() == "" { + klog.Warningf("skipping AddHostAlias for %q: empty/unspecified IP", name) + return nil + } + + // Exact line we want to ensure exists. + record := fmt.Sprintf("%s\t%s", ip.String(), name) + + // Fast path: if the exact line already exists, do nothing. + if _, err := c.RunCmd(exec.Command("grep", "-Fxq", record, "/etc/hosts")); err == nil { + return nil + } + + // Remove only existing lines for *this* name and *this* IP family, keep the other family. + dropRegex := hostAliasDropRegex(name, ip) + + if _, err := c.RunCmd(addHostAliasCommand(dropRegex, record, true, "/etc/hosts")); err != nil { + return errors.Wrap(err, "hosts update") + } + return nil } -func addHostAliasCommand(name string, record string, sudo bool, destPath string) *exec.Cmd { - sudoCmd := "sudo" - if !sudo { // for testing - sudoCmd = "" - } +// hostAliasDropRegex builds a regex that matches lines mapping the given name with the same IP family. +func hostAliasDropRegex(name string, ip net.IP) string { + qName := regexp.QuoteMeta(name) + if ip.To4() != nil { + // IPv4 lines like: 1.2.3.4 + return fmt.Sprintf(`^[[:space:]]*[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+[[:space:]]+%s$`, qName) + } + // IPv6 lines like: 2001:db8::1 (very permissive IPv6 matcher) + return fmt.Sprintf(`^[[:space:]]*[:0-9A-Fa-f]+[[:space:]]+%s$`, qName) +} - script := fmt.Sprintf( - `{ grep -v $'\t%s$' "%s"; echo "%s"; } > /tmp/h.$$; %s cp /tmp/h.$$ "%s"`, - name, - destPath, - record, - sudoCmd, - destPath) - return exec.Command("/bin/bash", "-c", script) +func addHostAliasCommand(dropRegex, record string, sudo bool, destPath string) *exec.Cmd { + sudoCmd := "sudo" + if !sudo { // for testing + sudoCmd = "" + } + + script := fmt.Sprintf( + `{ grep -v -F "%s" "%s"; echo "%s"; } > /tmp/h.$$; %s cp /tmp/h.$$ "%s"`, + record, + destPath, + record, + sudoCmd, + destPath) + return exec.Command("/bin/bash", "-c", script) } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 3325049811ec..1996b6a9e41c 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -631,12 +631,21 @@ func setupKubeconfig(h host.Host, cc config.ClusterConfig, n config.Node, cluste if hostIP, _, port, err = driver.ControlPlaneEndpoint(&cc, &n, h.DriverName); err != nil { exit.Message(reason.DrvCPEndpoint, fmt.Sprintf("failed to construct cluster server address: %v", err), out.V{"profileArg": fmt.Sprintf("--profile=%s", clusterName)}) } + + if hostIP == "" { + hostIP = "localhost" + } + } - addr := fmt.Sprintf("https://%s", net.JoinHostPort(hostIP, strconv.Itoa(port))) + + + // Build address *after* picking the final host (don’t string-replace inside a bracketed IPv6 literal). + hostForAddr := hostIP if cc.KubernetesConfig.APIServerName != constants.APIServerName { - addr = strings.ReplaceAll(addr, hostIP, cc.KubernetesConfig.APIServerName) - } + hostForAddr = cc.KubernetesConfig.APIServerName + } + addr := "https://" + net.JoinHostPort(hostForAddr, strconv.Itoa(port)) kcs := &kubeconfig.Settings{ ClusterName: clusterName, diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 89f5da5617fa..221c71bf572e 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -89,7 +89,10 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ExtraArgs: extraArgs, Network: cc.Network, Subnet: cc.Subnet, + Subnetv6: cc.Subnetv6, StaticIP: cc.StaticIP, + StaticIPv6: cc.StaticIPv6, + IPFamily: cc.KubernetesConfig.IPFamily, ListenAddress: cc.ListenAddress, GPUs: cc.GPUs, }), nil diff --git a/pkg/util/constants.go b/pkg/util/constants.go index db1a39c0568e..4fd1fab9dd1b 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -35,29 +35,49 @@ var DefaultAdmissionControllers = []string{ "ResourceQuota", } -// ServiceClusterIP returns the first IP of the ServiceCIDR +// ServiceClusterIP returns the first usable IP of the Service CIDR (network + 1) for either IPv4 or IPv6. func ServiceClusterIP(serviceCIDR string) (net.IP, error) { - ip, _, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return nil, errors.Wrap(err, "parsing default service cidr") - } - ip = ip.To4() - ip[3]++ - return ip, nil + base, _, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return nil, errors.Wrap(err, "parsing default service cidr") + } + ip := normalizeIP(base) + return addToIP(ip, 1), nil } -// DNSIP returns x.x.x.10 of the service CIDR func DNSIP(serviceCIDR string) (net.IP, error) { - ip, _, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return nil, errors.Wrap(err, "parsing default service cidr") - } - ip = ip.To4() - ip[3] = 10 - return ip, nil + base, _, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return nil, errors.Wrap(err, "parsing default service cidr") + } + ip := normalizeIP(base) + return addToIP(ip, 10), nil } // AlternateDNS returns a list of alternate names for a domain func AlternateDNS(domain string) []string { return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} } + + +// normalizeIP returns a 4-byte slice for v4 or 16-byte slice for v6. +func normalizeIP(ip net.IP) net.IP { + if v4 := ip.To4(); v4 != nil { + return v4 + } + return ip.To16() +} + +// addToIP returns ip + n, preserving length (v4/v6) with carry. +func addToIP(ip net.IP, n uint64) net.IP { + out := make(net.IP, len(ip)) + copy(out, ip) + i := len(out) - 1 + for n > 0 && i >= 0 { + sum := uint64(out[i]) + (n & 0xff) + out[i] = byte(sum & 0xff) + n = (n >> 8) + (sum >> 8) + i-- + } + return out +}