[BUG] yarn-copilot resource update failed on unrecognized node: hadoop-yarn-nm-2 about koordinator HOT 3 CLOSED

nm pod的yaml如下
apiVersion: v1
kind: Pod
metadata:
annotations:
cni.projectcalico.org/containerID: cd1d69000010b7f504918b6337f51073545cddd5b1aece4ea080c62c46c69932
cni.projectcalico.org/podIP: 10.244.168.31/32
cni.projectcalico.org/podIPs: 10.244.168.31/32
yarn.hadoop.apache.org/node-id: hadoop-yarn-nm-0:8041
creationTimestamp: "2024-06-06T13:12:53Z"
generateName: hadoop-yarn-nm-
labels:
app.kubernetes.io/component: node-manager
app.kubernetes.io/instance: hadoop-yarn
app.kubernetes.io/name: hadoop-yarn
controller-revision-hash: hadoop-yarn-nm-5b5cc8688c
koordinator.sh/qosClass: BE
statefulset.kubernetes.io/pod-name: hadoop-yarn-nm-0
name: hadoop-yarn-nm-0
namespace: hadoop-yarn
ownerReferences:

• apiVersion: apps/v1
  blockOwnerDeletion: true
  controller: true
  kind: StatefulSet
  name: hadoop-yarn-nm
  uid: bc2740be-043a-4068-af0a-5852059548f0
  resourceVersion: "1763244"
  uid: 5e73961d-1f00-461d-860e-7e33726b24f5
  spec:
  affinity:
  podAntiAffinity:
  requiredDuringSchedulingIgnoredDuringExecution:
  • labelSelector:
    matchLabels:
    app.kubernetes.io/component: node-manager
    app.kubernetes.io/instance: hadoop-yarn
    app.kubernetes.io/name: hadoop-yarn
    topologyKey: kubernetes.io/hostname
    containers:
• command:
  • /bin/bash
  • /tmp/hadoop-config/bootstrap.sh
  • -d
    env:
  • name: YARN_ROLE
    value: yarn-nm
  • name: NM_INIT_CPU_CORES
    value: "10"
  • name: NM_INIT_MEMORY_MB
    value: "10240"
  • name: POD_NAMESPACE
    valueFrom:
    fieldRef:
    apiVersion: v1
    fieldPath: metadata.namespace
  • name: POD_NAME
    valueFrom:
    fieldRef:
    apiVersion: v1
    fieldPath: metadata.name
    image: registry.cn-beijing.aliyuncs.com/koordinator-sh/apache-hadoop:3.3.3-v1.1
    imagePullPolicy: IfNotPresent
    livenessProbe:
    failureThreshold: 3
    httpGet:
    path: /node
    port: 8042
    scheme: HTTP
    initialDelaySeconds: 10
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 2
    name: yarn-nm
    readinessProbe:
    failureThreshold: 3
    httpGet:
    path: /node
    port: 8042
    scheme: HTTP
    initialDelaySeconds: 10
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 2
    resources:
    limits:
    kubernetes.io/batch-cpu: 2k
    kubernetes.io/batch-memory: 2Gi
    requests:
    kubernetes.io/batch-cpu: 1k
    kubernetes.io/batch-memory: 1Gi
    securityContext:
    privileged: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
  • mountPath: /tmp/hadoop-config
    name: hadoop-config
  • mountPath: /host-cgroup/
    name: host-cgroup-root
  • mountPath: /var/run/secrets/kubernetes.io/serviceaccount
    name: kube-api-access-8j9mw
    readOnly: true
• command:
  • /bin/bash
  • /tmp/hadoop-config/bootstrap.sh
  • -d
    env:
  • name: YARN_ROLE
    value: hdfs-dn
    image: registry.cn-beijing.aliyuncs.com/koordinator-sh/apache-hadoop:3.3.3-v1.1
    imagePullPolicy: IfNotPresent
    livenessProbe:
    failureThreshold: 3
    httpGet:
    path: /
    port: 9864
    scheme: HTTP
    initialDelaySeconds: 10
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 2
    name: hdfs-dn
    readinessProbe:
    failureThreshold: 3
    httpGet:
    path: /
    port: 8042
    scheme: HTTP
    initialDelaySeconds: 10
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 2
    resources:
    limits:
    kubernetes.io/batch-cpu: 1k
    kubernetes.io/batch-memory: 1Gi
    requests:
    kubernetes.io/batch-cpu: 1k
    kubernetes.io/batch-memory: 1Gi
    securityContext:
    privileged: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
  • mountPath: /tmp/hadoop-config
    name: hadoop-config
  • mountPath: /var/run/secrets/kubernetes.io/serviceaccount
    name: kube-api-access-8j9mw
    readOnly: true
    dnsPolicy: ClusterFirst
    enableServiceLinks: true
    hostname: hadoop-yarn-nm-0
    nodeName: 10.10.10.22
    priorityClassName: koord-batch
    restartPolicy: Always
    schedulerName: default-scheduler
    securityContext: {}
    serviceAccount: yarn-nodemanager
    serviceAccountName: yarn-nodemanager
    setHostnameAsFQDN: false
    subdomain: node-manager
    terminationGracePeriodSeconds: 0
    volumes:
• configMap:
  defaultMode: 420
  name: hadoop-yarn
  name: hadoop-config
• hostPath:
  path: /sys/fs/cgroup/
  type: ""
  name: host-cgroup-root
• name: kube-api-access-8j9mw
  projected:
  defaultMode: 420
  sources:
  • serviceAccountToken:
    expirationSeconds: 3607
    path: token
  • configMap:
    items:
    • key: ca.crt
      path: ca.crt
      name: kube-root-ca.crt
  • downwardAPI:
    items:
    • fieldRef:
      apiVersion: v1
      fieldPath: metadata.namespace
      path: namespace
      status:
      conditions:
• lastProbeTime: null
  lastTransitionTime: "2024-06-06T13:12:53Z"
  status: "True"
  type: Initialized
• lastProbeTime: null
  lastTransitionTime: "2024-06-06T13:13:13Z"
  status: "True"
  type: Ready
• lastProbeTime: null
  lastTransitionTime: "2024-06-06T13:13:13Z"
  status: "True"
  type: ContainersReady
• lastProbeTime: null
  lastTransitionTime: "2024-06-06T13:12:53Z"
  status: "True"
  type: PodScheduled
  containerStatuses:
• containerID: docker://9f129c606eafcb12c9bf07199b99b5c4884a64fa3fe78241e3d7d2004034a022
  image: registry.cn-beijing.aliyuncs.com/koordinator-sh/apache-hadoop:3.3.3-v1.1
  imageID: docker://sha256:d998b66c0037312d7b0740edbd8a49b1c746f52e52ec4578c05c4b0b966f79fc
  lastState: {}
  name: hdfs-dn
  ready: true
  restartCount: 0
  started: true
  state:
  running:
  startedAt: "2024-06-06T13:12:54Z"
• containerID: docker://21387e35289058b43ba0c87c7848beb61faa3d082f366c6099b711d46fce4df3
  image: registry.cn-beijing.aliyuncs.com/koordinator-sh/apache-hadoop:3.3.3-v1.1
  imageID: docker://sha256:d998b66c0037312d7b0740edbd8a49b1c746f52e52ec4578c05c4b0b966f79fc
  lastState: {}
  name: yarn-nm
  ready: true
  restartCount: 0
  started: true
  state:
  running:
  startedAt: "2024-06-06T13:12:54Z"
  hostIP: 10.10.10.22
  phase: Running
  podIP: 10.244.168.31
  podIPs:
• ip: 10.244.168.31
  qosClass: BestEffort
  startTime: "2024-06-06T13:12:53Z"

from koordinator.

the annotation yarn.hadoop.apache.org/node-id: hadoop-yarn-nm-0:8041 must be equal to node id in YARN.
this annotation is patched in start script in demo chart: https://github.com/koordinator-sh/charts/blob/main/hadoop-yarn/v3.3.3/templates/hadoop-configmap.yaml#L83

if you have changed the chart config such as nm pod template, remember to change this script also.

from koordinator.

是的，我改过一个配置，当时主机名申请67个字符报错超过64个字符，我就把setHostnameAsFQDN改成了false，然后就导致了这个问题。刚我把部署的namespace从hadoop-yarn改成yarn就没有报错了

from koordinator.