一文详解kubernetes 中资源分配的那些事

func addAllEventHandlers{
  ...
    informerFactory.Core().V1().Nodes().Informer().AddEventHandler(
      cache.ResourceEventHandlerFuncs{
        AddFunc:    sched.addNodeToCache,
        UpdateFunc: sched.updateNodeInCache,
        DeleteFunc: sched.deleteNodeFromCache,
      },
    )
  ...

func (sched *Scheduler) addNodeToCache(obj interface{}) {
  node, ok := obj.(*v1.Node)
  if !ok {
    klog.ErrorS(nil, "Cannot convert to *v1.Node", "obj", obj)
    return
  }
  nodeInfo := sched.Cache.AddNode(node)
  klog.V(3).InfoS("Add event for node", "node", klog.KObj(node))
  sched.SchedulingQueue.MoveAllToActiveOrBackoffQueue(queue.NodeAdd, preCheckForNode(nodeInfo))
}

func (p *PriorityQueue) MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) {
  p.lock.Lock()
  defer p.lock.Unlock()
  unschedulablePods := make([]*framework.QueuedPodInfo, 0, len(p.unschedulablePods.podInfoMap))
  for _, pInfo := range p.unschedulablePods.podInfoMap {
    if preCheck == nil || preCheck(pInfo.Pod) {
      unschedulablePods = append(unschedulablePods, pInfo)
    }
  }
  p.movePodsToActiveOrBackoffQueue(unschedulablePods, event)
}

func preCheckForNode(nodeInfo *framework.NodeInfo) queue.PreEnqueueCheck {
  // Note: the following checks doesn't take preemption into considerations, in very rare
  // cases (e.g., node resizing), "pod" may still fail a check but preemption helps. We deliberately
  // chose to ignore those cases as unschedulable pods will be re-queued eventually.
  return func(pod *v1.Pod) bool {
    admissionResults := AdmissionCheck(pod, nodeInfo, false)
    if len(admissionResults) != 0 {
      return false
    }
    _, isUntolerated := corev1helpers.FindMatchingUntoleratedTaint(nodeInfo.Node().Spec.Taints, pod.Spec.Tolerations, func(t *v1.Taint) bool {
      return t.Effect == v1.TaintEffectNoSchedule
    })
    return !isUntolerated
  }
}
func AdmissionCheck(pod *v1.Pod, nodeInfo *framework.NodeInfo, includeAllFailures bool) []AdmissionResult {
  var admissionResults []AdmissionResult
  insufficientResources := noderesources.Fits(pod, nodeInfo)
  if len(insufficientResources) != 0 {
    for i := range insufficientResources {
      admissionResults = append(admissionResults, AdmissionResult{InsufficientResource: &insufficientResources[i]})
    }
    if !includeAllFailures {
      return admissionResults
    }
  }
  if matches, _ := corev1nodeaffinity.GetRequiredNodeAffinity(pod).Match(nodeInfo.Node()); !matches {
    admissionResults = append(admissionResults, AdmissionResult{Name: nodeaffinity.Name, Reason: nodeaffinity.ErrReasonPod})
    if !includeAllFailures {
      return admissionResults
    }
}
  if !nodename.Fits(pod, nodeInfo) {
    admissionResults = append(admissionResults, AdmissionResult{Name: nodename.Name, Reason: nodename.ErrReason})
    if !includeAllFailures {
      return admissionResults
    }
  }
  if !nodeports.Fits(pod, nodeInfo) {
    admissionResults = append(admissionResults, AdmissionResult{Name: nodeports.Name, Reason: nodeports.ErrReason})
    if !includeAllFailures {
      return admissionResults
    }
  }
  return admissionResults
}

func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo) []InsufficientResource {
  return fitsRequest(computePodResourceRequest(pod), nodeInfo, nil, nil)
}
func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignoredExtendedResources, ignoredResourceGroups sets.String) []InsufficientResource {
  insufficientResources := make([]InsufficientResource, 0, 4)
  allowedPodNumber := nodeInfo.Allocatable.AllowedPodNumber
  if len(nodeInfo.Pods)+1 > allowedPodNumber {
    insufficientResources = append(insufficientResources, InsufficientResource{
      ResourceName: v1.ResourcePods,
      Reason:       "Too many pods",
      Requested:    1,
      Used:         int64(len(nodeInfo.Pods)),
      Capacity:     int64(allowedPodNumber),
    })
  }
  if podRequest.MilliCPU == 0 &&
    podRequest.Memory == 0 &&
    podRequest.EphemeralStorage == 0 &&
    len(podRequest.ScalarResources) == 0 {
    return insufficientResources
  }
  if podRequest.MilliCPU > (nodeInfo.Allocatable.MilliCPU - nodeInfo.Requested.MilliCPU) {
    insufficientResources = append(insufficientResources, InsufficientResource{
      ResourceName: v1.ResourceCPU,
      Reason:       "Insufficient cpu",
      Requested:    podRequest.MilliCPU,
      Used:         nodeInfo.Requested.MilliCPU,
      Capacity:     nodeInfo.Allocatable.MilliCPU,
    })
  }
  if podRequest.Memory > (nodeInfo.Allocatable.Memory - nodeInfo.Requested.Memory) {
    insufficientResources = append(insufficientResources, InsufficientResource{
      ResourceName: v1.ResourceMemory,
      Reason:       "Insufficient memory",
      Requested:    podRequest.Memory,
      Used:         nodeInfo.Requested.Memory,
      Capacity:     nodeInfo.Allocatable.Memory,
    })
}
  if podRequest.EphemeralStorage > (nodeInfo.Allocatable.EphemeralStorage - nodeInfo.Requested.EphemeralStorage) {
    insufficientResources = append(insufficientResources, InsufficientResource{
      ResourceName: v1.ResourceEphemeralStorage,
      Reason:       "Insufficient ephemeral-storage",
      Requested:    podRequest.EphemeralStorage,
      Used:         nodeInfo.Requested.EphemeralStorage,
      Capacity:     nodeInfo.Allocatable.EphemeralStorage,
    })
  }
  for rName, rQuant := range podRequest.ScalarResources {
    if v1helper.IsExtendedResourceName(rName) {
      // If this resource is one of the extended resources that should be ignored, we will skip checking it.
      // rName is guaranteed to have a slash due to api validation.
      var rNamePrefix string
      if ignoredResourceGroups.Len() > 0 {
        rNamePrefix = strings.Split(string(rName), "/")[0]
      }
      if ignoredExtendedResources.Has(string(rName)) || ignoredResourceGroups.Has(rNamePrefix) {
        continue
      }
    }
    if rQuant > (nodeInfo.Allocatable.ScalarResources[rName] - nodeInfo.Requested.ScalarResources[rName]) {
      insufficientResources = append(insufficientResources, InsufficientResource{
        ResourceName: rName,
        Reason:       fmt.Sprintf("Insufficient %v", rName),
        Requested:    podRequest.ScalarResources[rName],
        Used:         nodeInfo.Requested.ScalarResources[rName],
        Capacity:     nodeInfo.Allocatable.ScalarResources[rName],
      })
    }
  }
  return insufficientResources
}

kubectl describe node xxxx
Capacity:
 cpu:                64
 ephemeral-storage:  1056889268Ki
 hugepages-1Gi:      0
 hugepages-2Mi:      0
 memory:             263042696Ki
 pods:               110
Allocatable:
 cpu:                63
 ephemeral-storage:  1044306356Ki
 hugepages-1Gi:      0
 hugepages-2Mi:      0
 memory:             257799816Ki
 pods:               110

systemReserved:
  cpu: "0.5"
  ephemeral-storage: 1Gi
  memory: 2Gi
  pid: "1000"
kubeReserved:
  cpu: "0.5"
  ephemeral-storage: 1Gi
  memory: 2Gi
  pid: "1000
evictionHard:
  imagefs.available: 10Gi
  memory.available: 1Gi
  nodefs.available: 10Gi
  nodefs.inodesFree: 5%

# 该配置表示，capacity减去下面的配置的资源才是节点d当前可分配的
# 默认是pods，表示只减去pods占用了的资源
enforceNodeAllocatable:
- pods
- kube-reserved
- system-reserved
# 如果你使用的是systemd作为cgroup驱动，你还需要配置下面的配置
# 否则kubelet无法正常启动，因为找不到cgroup目录
# k8s官方推荐s会用systemd
kubeReservedCgroup: /kubelet.slice
systemReservedCgroup: /system.slice/kubelet.service