Changes to kernel module compression can break certain workflows about bottlerocket HOT 15 OPEN

###
# checking base_requirements ...
###
WARN: CONFIG_NET_CLS_BPF != y. Found m
WARN: CONFIG_NET_SCH_INGRESS != y. Found m
WARN: CONFIG_CRYPTO_USER_API_HASH != y. Found m
###
# checking itpables-based masq requirements ...
###
>>> passed all checks
###
# checking L7 and FQDN Policies requirements ...
###
>>> passed all checks
###
# checking ipsec requirements = y ...
###
>>> passed all checks
###
# checking ipsec requirements = m ...
###
WARN: CONFIG_INET_XFRM_MODE_TUNNEL != m. Found undefined
WARN: CONFIG_CRYPTO_AEAD != m. Found y
WARN: CONFIG_CRYPTO_AEAD2 != m. Found y
WARN: CONFIG_CRYPTO_SEQIV != m. Found y
WARN: CONFIG_CRYPTO_HMAC != m. Found y
WARN: CONFIG_CRYPTO_SHA256 != m. Found y
WARN: CONFIG_CRYPTO_AES != m. Found y
###
# checking bandwith_manager requirements ...
###
>>> passed all checks

apiVersion: v1
items:
- apiVersion: cilium.io/v2
  kind: CiliumClusterwideNetworkPolicy
  metadata:
    name: default
  spec:
    egress:
    - toEntities:
      - cluster
    - toEndpoints:
      - matchLabels:
          io.kubernetes.pod.namespace: kube-system
          k8s-app: kube-dns
      toPorts:
      - ports:
        - port: "53"
          protocol: UDP
        rules:
          dns:
          - matchPattern: '*'
    - toCIDRSet:
      - cidr: 0.0.0.0/0

apiVersion: v1
kind: Pod
metadata:
  name: my-pod
spec:
  containers:
  - name: my-container
    image: <my-image>
    securityContext:
      capabilities:
        add: ["SYS_MODULE"]
    volumeMounts:
    - name: kmod-static
      mountPath: /usr/local/sbin/modprobe
      readOnly: true
   - name: kernel-modules
      mountPath: /lib/modules/
      readOnly: true
  volumes:
    - name: kmod-static
      hostPath:
      path: /usr/bin/kmod
      type: File
    - name: kernel-modules
      hostPath:
      path: /lib/modules/

extraVolumeMounts:
  - name: kmod-static
    mountPath: /usr/local/sbin/modprobe
    readOnly: true
  - name: kernel-modules
    mountPath: /lib/modules/
    readOnly: true

extraVolumes:
  - name: kmod-static
    hostPath:
      path: /usr/bin/kmod
    type: File
  - name: kernel-modules
    hostPath:
      path: /lib/modules/

[2024-05-30T18:50:59Z INFO install_cni_chaining]: updating CNI conf file 10-aws.conflist -> 05-antrea.conflist
[2024-05-30T18:50:59Z INFO install_cni_chaining]: CNI conf file is already up-to-date
uid=0(root) gid=0(root) groups=0(root)
modprobe: ERROR: could not insert 'openvswitch': Operation not permitted
Failed to load the OVS kernel module from the container, try running 'modprobe openvswitch' on your Nodes

$ kubectl logs -n cilium cilium-sf6j6 | ag module
Defaulted container "cilium-agent" out of: cilium-agent, config (init), mount-cgroup (init), apply-sysctl-overwrites (init), mount-bpf-fs (init), clean-cilium-state (init), install-cni-binaries (init)
time="2024-06-07T11:24:29Z" level=warning msg="iptables modules could not be initialized. It probably means that iptables is not available on this system" error="could not load module iptable_raw: exit status 1" subsys=iptables

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: my-cilium-test
spec:
  selector:
    matchLabels:
      app: cilium
  template:
    metadata:
      labels:
        app: cilium
    spec:
      containers:
      - name: cilium-container
        image: cilium/cilium:stable
        command: ["/bin/sh", "-c"]
        args:
        - echo 'Starting cilium/cilium:stable Test';echo '---------------------------------------';if
            test -f /usr/local/sbin/modprobe; then echo 'Mounted correctly'; else exit 1;
            fi;if ! grep -E '^lru_cache' /proc/modules; then echo 'Adding a module'; modprobe
            lru_cache; grep -E '^lru_cache' /proc/modules && echo 'Successfully added a
            module'; fi;if grep -E '^lru_cache' /proc/modules; then echo 'Removing a module';
            modprobe -r lru_cache; grep -E '^lru_cache' /proc/modules || echo 'Successfully
            removed a module'; fi;exit 0
        securityContext:
          capabilities:
            add: ["SYS_MODULE"]
        volumeMounts:
        - name: kernel-modules
          mountPath: /lib/modules/
          readOnly: true
        - name: kmod-static
          mountPath: /usr/local/sbin/modprobe
          readOnly: true
      volumes:
      - hostPath:
          path: /lib/modules/
        name: kernel-modules
      - name: kmod-static
        hostPath:
          path: /usr/bin/kmod
          type: File

$ cilium -n cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             1 warnings
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

Deployment             hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1
Deployment             cilium-operator    Desired: 3, Ready: 3/3, Available: 3/3
DaemonSet              cilium             Desired: 15, Ready: 15/15, Available: 15/15
Deployment             hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1
Containers:            cilium-operator    Running: 3
                       hubble-ui          Running: 1
                       hubble-relay       Running: 1
                       cilium             Running: 15
Cluster Pods:          173/176 managed by Cilium
Helm chart version:    1.15.5
Image versions         hubble-relay       quay.io/cilium/hubble-relay:v1.15.5@sha256:1d24b24e3477ccf9b5ad081827db635419c136a2bd84a3e60f37b26a38dd0781: 1
                       cilium             quay.io/cilium/cilium:v1.15.5@sha256:4ce1666a73815101ec9a4d360af6c5b7f1193ab00d89b7124f8505dee147ca40: 15
                       cilium-operator    quay.io/cilium/operator-aws:v1.15.5@sha256:f9c0eaea023ce5a75b3ed1fc4b783f390c5a3c7dc1507a2dc4dbc667b80d1bd9: 3
                       hubble-ui          quay.io/cilium/hubble-ui:v0.13.0@sha256:7d663dc16538dd6e29061abd1047013a645e6e69c115e008bee9ea9fef9a6666: 1
                       hubble-ui          quay.io/cilium/hubble-ui-backend:v0.13.0@sha256:1e7657d997c5a48253bb8dc91ecee75b63018d16ff5e5797e5af367336bc8803: 1
Warnings:              cilium             cilium-q9b62    1 endpoints are not ready

$ k version
...
Server Version: v1.29.4-eks-036c24b

$ helm get values -n cilium cilium
USER-SUPPLIED VALUES:
USER-SUPPLIED VALUES: null
egressMasqueradeInterfaces: eth+
eni:
  awsReleaseExcessIPs: true
  enabled: true
extraVolumeMounts:
- mountPath: /usr/local/sbin/modprobe
  name: kmod-static
  readOnly: true
- mountPath: /lib/modules/
  name: kernel-modules
  readOnly: true
extraVolumes:
- hostPath:
    path: /usr/bin/kmod
  name: kmod-static
  type: File
- hostPath:
    path: /lib/modules/
  name: kernel-modules
gatewayAPI:
  enabled: true
hostServices:
  enabled: true
  protocols: tcp
hubble:
  enabled: true
  listenAddress: :4244
  metrics:
    enabled:
    - dns
    - drop
    - tcp
    - flow
    - port-distribution
    - icmp
    - http
  relay:
    enabled: true
    resources:
      limits:
        cpu: 100m
        memory: 48Mi
      requests:
        cpu: 50m
        memory: 24Mi
    rollOutPods: true
  ui:
    backend:
      resources:
        limits:
          cpu: 100m
          memory: 256Mi
        requests:
          cpu: 50m
          memory: 128Mi
    enabled: true
    frontend:
      resources:
        limits:
          cpu: 100m
          memory: 256Mi
        requests:
          cpu: 50m
          memory: 128Mi
    proxy:
      resources:
        limits:
          cpu: 100m
          memory: 256Mi
        requests:
          cpu: 50m
          memory: 128Mi
ingressController:
  enabled: true
  loadbalancerMode: dedicated
ipam:
  mode: eni
kubeProxyReplacement: partial
loadBalancer:
  l7:
    backend: envoy
nodePort:
  enabled: true
nodeinit:
  enabled: false
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
  restartPods: true
operator:
  nodeSelector:
    node.k8s.org/role: core
  prometheus:
    enabled: true
  replicas: 3
  resources:
    requests:
      cpu: 100m
      memory: 64Mi
  rollOutPods: true
policyEnforcementMode: always
prometheus:
  enabled: true
relay:
  nodeSelector:
    node.k8s.org/role: core
resources:
  requests:
    cpu: 100m
    memory: 192Mi
rollOutCiliumPods: true
tunnel: disabled
ui:
  nodeSelector:
    node.k8s.org/role: core

$ k describe no my-node
...
Non-terminated Pods:          (6 in total)
  Namespace                   Name                                         CPU Requests  CPU Limits  Memory Requests  Memory Limits  Age
  ---------                   ----                                         ------------  ----------  ---------------  -------------  ---
  cilium                      cilium-wpgrt                                 100m (6%)     0 (0%)      192Mi (7%)       0 (0%)         4d18h
  kube-system                 kube-proxy-hh6nk                             100m (6%)     0 (0%)      0 (0%)           0 (0%)         4d18h
  monitoring                  prometheus-prometheus-node-exporter-q27ll    0 (0%)        0 (0%)      0 (0%)           0 (0%)         4d18h
  operators                   ebs-csi-node-jgls7                           30m (1%)      0 (0%)      120Mi (4%)       768Mi (29%)    4d18h
  operators                   efs-csi-node-ls8bz                           100m (6%)     0 (0%)      128Mi (4%)       0 (0%)         4d18h
  operators                   karpenter-657c4785d6-2d588                   0 (0%)        0 (0%)      0 (0%)           0 (0%)         4d17h

$ k get node -o yaml my-node
...
spec:
  taints:
  - effect: NoSchedule
    key: karpenter

$ cat bottlerocket-userdata.tpl
# https://github.com/bottlerocket-os/bottlerocket/blob/develop/README.md#description-of-settings
[settings.kubernetes]
cluster-name = "${cluster_name}"
api-server = "${cluster_endpoint}"
cluster-certificate = "${cluster_ca_data}"

# Hardening based on https://github.com/bottlerocket-os/bottlerocket/blob/develop/SECURITY_GUIDANCE.md

# Enable kernel lockdown in "integrity" mode.
# This prevents modifications to the running kernel, even by privileged users.
[settings.kernel]
lockdown = "integrity"

# The admin host container provides SSH access and runs with "superpowers".
# It is disabled by default, but can be disabled explicitly.
[settings.host-containers.admin]
enabled = ${enable_admin_container}
superpowered = true

# The control host container provides out-of-band access via SSM.
# It is enabled by default, and can be disabled if you do not expect to use SSM.
# This could leave you with no way to access the API and change settings on an existing node!
[settings.host-containers.control]
enabled = ${enable_control_container}
superpowered = true

# Eviction settings
[settings.kubernetes.eviction-hard]
"memory.available" = "200Mi"

# Reservation settings
[settings.kubernetes.system-reserved]
cpu = "200m"
memory = "500Mi"
ephemeral-storage= "1Gi"

# Kubelet reservation
[settings.kubernetes.kube-reserved]
cpu = "200m"
memory = "550Mi"
ephemeral-storage= "1Gi"

# Strip unneeded kernel modules
[settings.kernel.modules.sctp]
allowed = false

[settings.kernel.modules.udf]
allowed = false

[settings.kernel.modules.rds]
allowed = false

[settings.kernel.modules.dccp]
allowed = false

[settings.kernel.modules.jffs2]
allowed = false

[settings.kernel.modules.cramfs]
allowed = false

[settings.kernel.modules.freevxfs]
allowed = false

[settings.kernel.modules.hfs]
allowed = false

[settings.kernel.modules.hfsplus]
allowed = false

[settings.kernel.modules.squashfs]
allowed = false

[settings.kernel.modules.vfat]
allowed = false

[settings.kernel.modules.usb-storage]
allowed = false

[settings.kernel.modules.tipc]
allowed = false

# Sysctl tuning
[settings.kernel.sysctl]
"vm.swappiness" = "0"
"net.ipv4.tcp_syncookies" = "1"
"net.ipv4.ip_local_port_range" = "1024 65535"
"net.ipv4.tcp_tw_recycle" = "1"
"net.ipv4.tcp_fin_timeout" = "15"
"net.core.somaxconn" = "4096"
"net.core.netdev_max_backlog" = "4096"
"net.core.rmem_max" = "16777216"
"net.core.wmem_max" = "16777216"
"net.ipv4.tcp_max_syn_backlog" = "20480"
"net.ipv4.tcp_no_metrics_save" = "1"
"net.ipv4.tcp_rmem" = "4096 87380 16777216"
"net.ipv4.tcp_syn_retries" = "2"
"net.ipv4.tcp_synack_retries" = "2"
"net.ipv4.tcp_wmem" = "4096 65536 16777216"
"net.ipv4.tcp_max_tw_buckets" = "1440000"
"net.core.default_qdisc" = "fq_codel"
"net.ipv4.tcp_congestion_control" = "bbr"
${more_options}

# Add some labels
[settings.kubernetes.node-labels]
${node_labels}

    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

Deployment             hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1
Deployment             cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
DaemonSet              cilium             Desired: 2, Ready: 2/2, Available: 2/2
Containers:            cilium             Running: 2
                       hubble-relay       Running: 1
                       cilium-operator    Running: 1
Cluster Pods:          7/7 managed by Cilium
Helm chart version:
Image versions         cilium             quay.io/cilium/cilium:v1.15.6@sha256:6aa840986a3a9722cd967ef63248d675a87add7e1704740902d5d3162f0c0def: 2
                       hubble-relay       quay.io/cilium/hubble-relay:v1.15.6@sha256:a0863dd70d081b273b87b9b7ce7e2d3f99171c2f5e202cd57bc6691e51283e0c: 1
                       cilium-operator    quay.io/cilium/operator-aws:v1.15.6@sha256:9656d44ee69817d156cc7d3797f92de2e534dfb991610c79c00e097b4dedd620: 1

root@ip:/home/cilium# cat /proc/mounts | grep modprobe
/dev/root /usr/local/sbin/modprobe ext4 ro,seclabel,relatime,stripe=1024 0 0

root@ip:/home/cilium# ldd /usr/local/sbin/modprobe
	not a dynamic executable