Kubernetes教程(十六)---从 Service DNS 记录到 IP 地址，KubeDNS 工作原理

[root@test ~]# kubectl -n kube-system get po
NAME                                       READY   STATUS    RESTARTS        AGE
calico-kube-controllers-6c557bcb96-kxlp8   1/1     Running   0               5d19h
calico-node-rd28x                          1/1     Running   0               5d19h
coredns-84798fc4bb-8rrzz                   1/1     Running   0               4d22h
coredns-84798fc4bb-sbtdr                   1/1     Running   0               4d22h
etcd-test                                  1/1     Running   0               5d19h
kc-kubectl-7f854bbd9b-mfdl7                1/1     Running   0               5d19h
kube-apiserver-test                        1/1     Running   0               5d19h
kube-controller-manager-test               1/1     Running   9 (2d12h ago)   5d19h
kube-proxy-2zcch                           1/1     Running   0               5d19h
kube-scheduler-test                        1/1     Running   9 (2d12h ago)   5d19h

[root@test ~]# kubectl -n kube-system get cm
NAME                                 DATA   AGE
calico-config                        4      5d19h
coredns                              1      5d19h
extension-apiserver-authentication   6      5d19h
kube-proxy                           2      5d19h
kube-root-ca.crt                     1      5d19h
kubeadm-config                       1      5d19h
kubelet-config-1.23                  1      5d19h

[root@test ~]# kubectl -n kube-system get cm  coredns -oyaml
apiVersion: v1
data:
  Corefile: |-
    .:53 {
        errors
        health {
            lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
            pods insecure
            fallthrough in-addr.arpa ip6.arpa
            ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf
        cache 10
        loop
        reload 10s
        loadbalance
    }
kind: ConfigMap
metadata:
  creationTimestamp: "2023-02-27T06:47:05Z"
  name: coredns
  namespace: kube-system
  resourceVersion: "187919"
  uid: 76f7a78f-70ce-4074-9660-aa317cc7c1c1

// plugin/kubernetes/setup.go#33
func setup(c *caddy.Controller) error {
   // Do not call klog.InitFlags(nil) here.  It will cause reload to panic.
   klog.SetOutput(os.Stdout)
   // 检查了 corefile 中 kubernetes 配置的定义，并配置了一些缺省值
   k, err := kubernetesParse(c)
   if err != nil {
      return plugin.Error(pluginName, err)
   }
    // 启动了对 pod, service, endpoint 三种资源增、删、改的 watch，并注册了一些回调
    // 注意：pod 是否启动 watch 是根据配置文件中 pod 的值来决定的，如果值不是 verified 就不会启动 pod 的 watch
    // 这里的 watch 方法观测到变化后，仅仅只改变 dns.modified 这个值，它会将该值设置为当前时间戳
   onStart, onShut, err := k.InitKubeCache(context.Background())
   if err != nil {
      return plugin.Error(pluginName, err)
   }
   if onStart != nil {
      c.OnStartup(onStart)
   }
   if onShut != nil {
      c.OnShutdown(onShut)
   }
   // 添加插件到插件链里
   dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
      k.Next = next
      return k
   })

   // get locally bound addresses
   c.OnStartup(func() error {
      k.localIPs = boundIPs(c)
      return nil
   })

   return nil
}

//plugin/kubernetes/controller.go#563
func (dns *dnsControl) Add(obj interface{})               { dns.updateModified() }
func (dns *dnsControl) Delete(obj interface{})            { dns.updateModified() }
func (dns *dnsControl) Update(oldObj, newObj interface{}) { dns.detectChanges(oldObj, newObj) }

// updateModified set dns.modified to the current time.
func (dns *dnsControl) updateModified() {
   unix := time.Now().Unix()
   atomic.StoreInt64(&dns.modified, unix)
}

// updateExtModified set dns.extModified to the current time.
func (dns *dnsControl) updateExtModifed() {
   unix := time.Now().Unix()
   atomic.StoreInt64(&dns.extModified, unix)
}

// detectChanges detects changes in objects, and updates the modified timestamp
func (dns *dnsControl) detectChanges(oldObj, newObj interface{}) {
   // If both objects have the same resource version, they are identical.
   if newObj != nil && oldObj != nil && (oldObj.(meta.Object).GetResourceVersion() == newObj.(meta.Object).GetResourceVersion()) {
      return
   }
   obj := newObj
   if obj == nil {
      obj = oldObj
   }
   switch ob := obj.(type) {
   case *object.Service:
      imod, emod := serviceModified(oldObj, newObj)
      if imod {
         dns.updateModified()
      }
      if emod {
         dns.updateExtModifed()
      }
   case *object.Pod:
      dns.updateModified()
   case *object.Endpoints:
      if !endpointsEquivalent(oldObj.(*object.Endpoints), newObj.(*object.Endpoints)) {
         dns.updateModified()
      }
   default:
      log.Warningf("Updates for %T not supported.", ob)
   }
}

// plugin/kubernetes/kubernetes.go#95
func (k *Kubernetes) Services(ctx context.Context, state request.Request, exact bool, opt plugin.Options) (svcs []msg.Service, err error) {
   // We're looking again at types, which we've already done in ServeDNS, but there are some types k8s just can't answer.
   switch state.QType() {
   // 如果是插件 DNS 服务版本的话就返回编译时的版本号
   case dns.TypeTXT:
      // 1 label + zone, label must be "dns-version".
      t, _ := dnsutil.TrimZone(state.Name(), state.Zone)

      segs := dns.SplitDomainName(t)
      if len(segs) != 1 {
         return nil, nil
      }
      if segs[0] != "dns-version" {
         return nil, nil
      }
      svc := msg.Service{Text: DNSSchemaVersion, TTL: 28800, Key: msg.Path(state.QName(), coredns)}
      return []msg.Service{svc}, nil
   // 如果查询的是 NS 类型记录，就返回 CoreDNS service 自身在集群里的记录
   case dns.TypeNS:
      // We can only get here if the qname equals the zone, see ServeDNS in handler.go.
      nss := k.nsAddrs(false, state.Zone)
      var svcs []msg.Service
      for _, ns := range nss {
         if ns.Header().Rrtype == dns.TypeA {
            svcs = append(svcs, msg.Service{Host: ns.(*dns.A).A.String(), Key: msg.Path(ns.Header().Name, coredns), TTL: k.ttl})
            continue
         }
         if ns.Header().Rrtype == dns.TypeAAAA {
            svcs = append(svcs, msg.Service{Host: ns.(*dns.AAAA).AAAA.String(), Key: msg.Path(ns.Header().Name, coredns), TTL: k.ttl})
         }
      }
      return svcs, nil
   }

   if isDefaultNS(state.Name(), state.Zone) {
      nss := k.nsAddrs(false, state.Zone)
      var svcs []msg.Service
      for _, ns := range nss {
         if ns.Header().Rrtype == dns.TypeA && state.QType() == dns.TypeA {
            svcs = append(svcs, msg.Service{Host: ns.(*dns.A).A.String(), Key: msg.Path(state.QName(), coredns), TTL: k.ttl})
            continue
         }
         if ns.Header().Rrtype == dns.TypeAAAA && state.QType() == dns.TypeAAAA {
            svcs = append(svcs, msg.Service{Host: ns.(*dns.AAAA).AAAA.String(), Key: msg.Path(state.QName(), coredns), TTL: k.ttl})
         }
      }
      return svcs, nil
   }
   // 到这里才正式开始解析 DNS 请求
   s, e := k.Records(ctx, state, false)

   // SRV for external services is not yet implemented, so remove those records.

   if state.QType() != dns.TypeSRV {
      return s, e
   }

   internal := []msg.Service{}
   for _, svc := range s {
      if t, _ := svc.HostType(); t != dns.TypeCNAME {
         internal = append(internal, svc)
      }
   }

   return internal, e
}

// plugin/kubernetes/kubernetes.go#370
func (k *Kubernetes) Records(ctx context.Context, state request.Request, exact bool) ([]msg.Service, error) {
   // 将请求数据解析为service、namespace 之类的格式
   r, e := parseRequest(state.Name(), state.Zone)
   if e != nil {
      return nil, e
   }
   // 数据异常直接返回
   if r.podOrSvc == "" {
      return nil, nil
   }

   if dnsutil.IsReverse(state.Name()) > 0 {
      return nil, errNoItems
   }

   if !k.namespaceExposed(r.namespace) {
      return nil, errNsNotExposed
   }
   // 查询 pod 
   if r.podOrSvc == Pod {
      pods, err := k.findPods(r, state.Zone)
      return pods, err
   }
   // 查询 svc
   services, err := k.findServices(r, state.Zone)
   return services, err
}

// plugin/kubernetes/kubernetes.go#412
func (k *Kubernetes) findPods(r recordRequest, zone string) (pods []msg.Service, err error) {
   if k.podMode == podModeDisabled {
      return nil, errNoItems
   }

   namespace := r.namespace
   if !k.namespaceExposed(namespace) {
      return nil, errNoItems
   }

   podname := r.service

   // handle empty pod name
   if podname == "" {
      if k.namespaceExposed(namespace) {
         // NODATA
         return nil, nil
      }
      // NXDOMAIN
      return nil, errNoItems
   }

   zonePath := msg.Path(zone, coredns)
   ip := ""
   if strings.Count(podname, "-") == 3 && !strings.Contains(podname, "--") {
      ip = strings.ReplaceAll(podname, "-", ".")
   } else {
      ip = strings.ReplaceAll(podname, "-", ":")
   }

   if k.podMode == podModeInsecure {
      if !k.namespaceExposed(namespace) { // namespace does not exist
         return nil, errNoItems
      }

      // If ip does not parse as an IP address, we return an error, otherwise we assume a CNAME and will try to resolve it in backend_lookup.go
      if net.ParseIP(ip) == nil {
         return nil, errNoItems
      }

      return []msg.Service{{Key: strings.Join([]string{zonePath, Pod, namespace, podname}, "/"), Host: ip, TTL: k.ttl}}, err
   }

   // PodModeVerified
   err = errNoItems

   for _, p := range k.APIConn.PodIndex(ip) {
      // check for matching ip and namespace
      if ip == p.PodIP && match(namespace, p.Namespace) {
         s := msg.Service{Key: strings.Join([]string{zonePath, Pod, namespace, podname}, "/"), Host: ip, TTL: k.ttl}
         pods = append(pods, s)

         err = nil
      }
   }
   return pods, err
}

// findServices returns the services matching r from the cache.
func (k *Kubernetes) findServices(r recordRequest, zone string) (services []msg.Service, err error) {
   if !k.namespaceExposed(r.namespace) {
      return nil, errNoItems
   }

   // handle empty service name
   if r.service == "" {
      if k.namespaceExposed(r.namespace) {
         // NODATA
         return nil, nil
      }
      // NXDOMAIN
      return nil, errNoItems
   }

   err = errNoItems

   var (
      endpointsListFunc func() []*object.Endpoints
      endpointsList     []*object.Endpoints
      serviceList       []*object.Service
   )

   idx := object.ServiceKey(r.service, r.namespace)
   serviceList = k.APIConn.SvcIndex(idx)
   endpointsListFunc = func() []*object.Endpoints { return k.APIConn.EpIndex(idx) }

   zonePath := msg.Path(zone, coredns)
   for _, svc := range serviceList {
      if !(match(r.namespace, svc.Namespace) && match(r.service, svc.Name)) {
         continue
      }

      // If "ignore empty_service" option is set and no endpoints exist, return NXDOMAIN unless
      // it's a headless or externalName service (covered below).
      if k.opts.ignoreEmptyService && svc.Type != api.ServiceTypeExternalName && !svc.Headless() { // serve NXDOMAIN if no endpoint is able to answer
         podsCount := 0
         for _, ep := range endpointsListFunc() {
            for _, eps := range ep.Subsets {
               podsCount += len(eps.Addresses)
            }
         }

         if podsCount == 0 {
            continue
         }
      }

      // External service
      if svc.Type == api.ServiceTypeExternalName {
         s := msg.Service{Key: strings.Join([]string{zonePath, Svc, svc.Namespace, svc.Name}, "/"), Host: svc.ExternalName, TTL: k.ttl}
         if t, _ := s.HostType(); t == dns.TypeCNAME {
            s.Key = strings.Join([]string{zonePath, Svc, svc.Namespace, svc.Name}, "/")
            services = append(services, s)

            err = nil
         }
         continue
      }

      // Endpoint query or headless service
      if svc.Headless() || r.endpoint != "" {
         if endpointsList == nil {
            endpointsList = endpointsListFunc()
         }

         for _, ep := range endpointsList {
            if object.EndpointsKey(svc.Name, svc.Namespace) != ep.Index {
               continue
            }

            for _, eps := range ep.Subsets {
               for _, addr := range eps.Addresses {

                  // See comments in parse.go parseRequest about the endpoint handling.
                  if r.endpoint != "" {
                     if !match(r.endpoint, endpointHostname(addr, k.endpointNameMode)) {
                        continue
                     }
                  }

                  for _, p := range eps.Ports {
                     if !(matchPortAndProtocol(r.port, p.Name, r.protocol, p.Protocol)) {
                        continue
                     }
                     s := msg.Service{Host: addr.IP, Port: int(p.Port), TTL: k.ttl}
                     s.Key = strings.Join([]string{zonePath, Svc, svc.Namespace, svc.Name, endpointHostname(addr, k.endpointNameMode)}, "/")

                     err = nil

                     services = append(services, s)
                  }
               }
            }
         }
         continue
      }

      // ClusterIP service
      for _, p := range svc.Ports {
         if !(matchPortAndProtocol(r.port, p.Name, r.protocol, string(p.Protocol))) {
            continue
         }

         err = nil

         for _, ip := range svc.ClusterIPs {
            s := msg.Service{Host: ip, Port: int(p.Port), TTL: k.ttl}
            s.Key = strings.Join([]string{zonePath, Svc, svc.Namespace, svc.Name}, "/")
            services = append(services, s)
         }
      }
   }
   return services, err
}

func (dns *dnsControl) SvcIndex(idx string) (svcs []*object.Service) {
   os, err := dns.svcLister.ByIndex(svcNameNamespaceIndex, idx)
   if err != nil {
      return nil
   }
   for _, o := range os {
      s, ok := o.(*object.Service)
      if !ok {
         continue
      }
      svcs = append(svcs, s)
   }
   return svcs
}

kubectl create deployment nginx --image nginx

[root@test ~]# k get po
NAME                     READY   STATUS    RESTARTS   AGE
nginx-85b98978db-mttqh   1/1     Running   0          77s
[root@test ~]# kubectl exec -it nginx-85b98978db-mttqh -- cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local
nameserver 10.96.0.10
options ndots:5

[root@test ~]# kubectl -n kube-system get svc kube-dns
NAME       TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                                    AGE
kube-dns   ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP,9153/TCP   5d20h

apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: dns-example
spec:
  containers:
    - name: test
      image: nginx
  dnsPolicy: "None"
  dnsConfig:
    nameservers:
      - 192.0.2.1 # 这是一个示例
    searches:
      - ns1.svc.cluster-domain.example
      - my.dns.search.suffix
    options:
      - name: ndots
        value: "2"
      - name: edns0

// pkg/kubelet/network/dns/dns.go#L384
// GetPodDNS returns DNS settings for the pod.
func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
	dnsConfig, err := c.getHostDNSConfig(c.ResolverConfig)
	if err != nil {
		return nil, err
	}
	// 首先获取类型，然后根据不同的配置，有不同的逻辑
	dnsType, err := getPodDNSType(pod)
	if err != nil {
		klog.ErrorS(err, "Failed to get DNS type for pod. Falling back to DNSClusterFirst policy.", "pod", klog.KObj(pod))
		dnsType = podDNSCluster // 没有指定就使用 DNSClusterFirst
	}
	switch dnsType {
	case podDNSNone:
		// DNSNone should use empty DNS settings as the base.
		dnsConfig = &runtimeapi.DNSConfig{}
	case podDNSCluster:
		if len(c.clusterDNS) != 0 {
			// For a pod with DNSClusterFirst policy, the cluster DNS server is
			// the only nameserver configured for the pod. The cluster DNS server
			// itself will forward queries to other nameservers that is configured
			// to use, in case the cluster DNS server cannot resolve the DNS query
			// itself.
			dnsConfig.Servers = []string{}
			for _, ip := range c.clusterDNS {
				dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
			}
			dnsConfig.Searches = c.generateSearchesForDNSClusterFirst(dnsConfig.Searches, pod)
			dnsConfig.Options = defaultDNSOptions
			break
		}
		// clusterDNS is not known. Pod with ClusterDNSFirst Policy cannot be created.
		nodeErrorMsg := fmt.Sprintf("kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. Falling back to %q policy.", v1.DNSClusterFirst, v1.DNSDefault)
		c.recorder.Eventf(c.nodeRef, v1.EventTypeWarning, "MissingClusterDNS", nodeErrorMsg)
		c.recorder.Eventf(pod, v1.EventTypeWarning, "MissingClusterDNS", "pod: %q. %s", format.Pod(pod), nodeErrorMsg)
		// Fallback to DNSDefault.
		fallthrough
	case podDNSHost:
		// When the kubelet --resolv-conf flag is set to the empty string, use
		// DNS settings that override the docker default (which is to use
		// /etc/resolv.conf) and effectively disable DNS lookups. According to
		// the bind documentation, the behavior of the DNS client library when
		// "nameservers" are not specified is to "use the nameserver on the
		// local machine". A nameserver setting of localhost is equivalent to
		// this documented behavior.
		if c.ResolverConfig == "" {
			for _, nodeIP := range c.nodeIPs {
				if utilnet.IsIPv6(nodeIP) {
					dnsConfig.Servers = append(dnsConfig.Servers, "::1")
				} else {
					dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
				}
			}
			if len(dnsConfig.Servers) == 0 {
				dnsConfig.Servers = append(dnsConfig.Servers, "127.0.0.1")
			}
			dnsConfig.Searches = []string{"."}
		}
	}

	if pod.Spec.DNSConfig != nil {
		dnsConfig = appendDNSConfig(dnsConfig, pod.Spec.DNSConfig)
	}
	return c.formDNSConfigFitsLimits(dnsConfig, pod), nil
}

	dnsType, err := getPodDNSType(pod)
	if err != nil {
		klog.ErrorS(err, "Failed to get DNS type for pod. Falling back to DNSClusterFirst policy.", "pod", klog.KObj(pod))
		dnsType = podDNSCluster // 
	}

			dnsConfig.Servers = []string{}
			for _, ip := range c.clusterDNS {
				dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
			}
			dnsConfig.Searches = c.generateSearchesForDNSClusterFirst(dnsConfig.Searches, pod)
			dnsConfig.Options = defaultDNSOptions

  dnsConfig.Servers = []string{}
  for _, ip := range c.clusterDNS {
    dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
  }

[root@test ~]# cat /var/lib/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
kind: KubeletConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 3s
volumeStatsAggPeriod: 1m0s

// pkg/kubelet/network/dns/dns.go#L164
func (c *Configurer) generateSearchesForDNSClusterFirst(hostSearch []string, pod *v1.Pod) []string {
	if c.ClusterDomain == "" {
		return hostSearch
	}

	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, c.ClusterDomain)
	svcDomain := fmt.Sprintf("svc.%s", c.ClusterDomain)
	clusterSearch := []string{nsSvcDomain, svcDomain, c.ClusterDomain}

	return omitDuplicates(append(clusterSearch, hostSearch...))
}

var (
	// The default dns opt strings.
	defaultDNSOptions = []string{"ndots:5"}
)

// pkg/kubelet/network/dns/dns.go#430
// SetupDNSinContainerizedMounter replaces the nameserver in containerized-mounter's rootfs/etc/resolv.conf with kubelet.ClusterDNS
func (c *Configurer) SetupDNSinContainerizedMounter(mounterPath string) {
   resolvePath := filepath.Join(strings.TrimSuffix(mounterPath, "/mounter"), "rootfs", "etc", "resolv.conf")
   dnsString := ""
   for _, dns := range c.clusterDNS {
      dnsString = dnsString + fmt.Sprintf("nameserver %s\n", dns)
   }
   if c.ResolverConfig != "" {
      f, err := os.Open(c.ResolverConfig)
      if err != nil {
         klog.ErrorS(err, "Could not open resolverConf file")
      } else {
         defer f.Close()
         _, hostSearch, _, err := parseResolvConf(f)
         if err != nil {
            klog.ErrorS(err, "Error for parsing the resolv.conf file")
         } else {
            dnsString = dnsString + "search"
            for _, search := range hostSearch {
               dnsString = dnsString + fmt.Sprintf(" %s", search)
            }
            dnsString = dnsString + "\n"
         }
      }
   }
   if err := ioutil.WriteFile(resolvePath, []byte(dnsString), 0600); err != nil {
      klog.ErrorS(err, "Could not write dns nameserver in the file", "path", resolvePath)
   }
}

// cmd/kubeadm/app/componentconfigs/kubelet.go#129
func (kc *kubeletConfig) Default(cfg *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) {
   const kind = "KubeletConfiguration"

   if kc.config.FeatureGates == nil {
      kc.config.FeatureGates = map[string]bool{}
   }

   if kc.config.StaticPodPath == "" {
      kc.config.StaticPodPath = kubeadmapiv1.DefaultManifestsDir
   } else if kc.config.StaticPodPath != kubeadmapiv1.DefaultManifestsDir {
      warnDefaultComponentConfigValue(kind, "staticPodPath", kubeadmapiv1.DefaultManifestsDir, kc.config.StaticPodPath)
   }

   clusterDNS := ""
   dnsIP, err := constants.GetDNSIP(cfg.Networking.ServiceSubnet)
   if err != nil {
      clusterDNS = kubeadmapiv1.DefaultClusterDNSIP
   } else {
      clusterDNS = dnsIP.String()
   }

   if kc.config.ClusterDNS == nil {
      kc.config.ClusterDNS = []string{clusterDNS}
   } else if len(kc.config.ClusterDNS) != 1 || kc.config.ClusterDNS[0] != clusterDNS {
      warnDefaultComponentConfigValue(kind, "clusterDNS", []string{clusterDNS}, kc.config.ClusterDNS)
   }
   
   if kc.config.ClusterDomain == "" {
   kc.config.ClusterDomain = cfg.Networking.DNSDomain
   } else if cfg.Networking.DNSDomain != "" && kc.config.ClusterDomain != cfg.Networking.DNSDomain {
   warnDefaultComponentConfigValue(kind, "clusterDomain", cfg.Networking.DNSDomain, kc.config.ClusterDomain)
}
   // ....省略 
 }

DefaultServicesSubnet = "10.96.0.0/12"
DefaultClusterDNSIP = "10.96.0.10"