openshift / lvm-operator Goto Github PK

[root@master-0 core]# vgs
  VG  #PV #LV #SN Attr   VSize VFree 
  vg1   5   4   0 wz--n- 6.00t <6.00t
[root@master-0 core]# pvs
  PV           VG  Fmt  Attr PSize   PFree  
  /dev/nvme0n1 vg1 lvm2 a--  745.21g 745.21g
  /dev/nvme1n1 vg1 lvm2 a--  745.21g 745.21g
  /dev/sda     vg1 lvm2 a--   <2.73t   2.72t
  /dev/sdc     vg1 lvm2 a--  931.48g 931.48g
  /dev/sde     vg1 lvm2 a--  931.48g 931.48g

$ oc get lvmvolumegroupnodestatuses -o yaml
apiVersion: v1
items:
- apiVersion: lvm.topolvm.io/v1alpha1
  kind: LVMVolumeGroupNodeStatus
  metadata:
    creationTimestamp: "2022-02-05T18:58:01Z"
    generation: 1
    name: master-0.apollo2.hpecloud.org
    namespace: lvm-operator-system
    resourceVersion: "6276721"
    uid: b6cc0395-e17c-4116-941f-9b95b8c4ed82
  spec:
    nodeStatus:
    - devices:
      - /dev/nvme0n1
      - /dev/nvme1n1
      - /dev/sda
      name: vg1
      status: Ready
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

$ oc get lvmcluster  lvmcluster-sample -o yaml
apiVersion: lvm.topolvm.io/v1alpha1
kind: LVMCluster
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"lvm.topolvm.io/v1alpha1","kind":"LVMCluster","metadata":{"annotations":{},"name":"lvmcluster-sample","namespace":"lvm-operator-system"},"spec":{"deviceClasses":[{"name":"vg1"}]}}
  creationTimestamp: "2022-02-05T18:56:54Z"
  finalizers:
  - lvmcluster.topolvm.io
  generation: 1
  name: lvmcluster-sample
  namespace: lvm-operator-system
  resourceVersion: "6276722"
  uid: fb91728e-1300-4fe1-9041-78d5a8e166d6
spec:
  deviceClasses:
  - name: vg1
status:
  deviceClassStatuses:
  - name: vg1
    nodeStatus:
    - devices:
      - /dev/nvme0n1
      - /dev/nvme1n1
      - /dev/sda
      node: master-0.apollo2.hpecloud.org
      status: Ready
  ready: true

output rules (optionally as output:<generator>:...)

+output:artifacts[:code=<string>],config=<string>  package  outputs artifacts to different locations, depending on whether they're package-associated or not.   
+output:dir=<string>                               package  outputs each artifact to the given directory, regardless of if it's package-associated or not.      
+output:none                                       package  skips outputting anything.                                                                          
+output:stdout                                     package  outputs everything to standard-out, with no separation.                                             

run `controller-gen rbac:roleName=manager-role crd webhook paths=./... output:crd:artifacts:config=config/crd/bases -w` to see all available markers, or `controller-gen rbac:roleName=manager-role crd webhook paths=./... output:crd:artifacts:config=config/crd/bases -h` for usage
make: *** [Makefile:92: manifests] Error 1

$ oc get pods
NAME                                  READY   STATUS     RESTARTS   AGE
controller-manager-66b84d759f-mpg5p   3/3     Running    0          6m7s
topolvm-controller-df459cdd5-pfkjb    4/4     Running    0          5m50s
topolvm-node-fd92q                    0/4     Init:0/1   0          5m50s
vg-manager-2mxdr                      1/1     Running    0          5m50s

root       31332  0.0  0.0  11920  2888 ?        Ss   18:29   0:00 sh -c until [ -f /etc/topolvm/lvmd.yaml ]; do echo waiting for lvmd config file; sleep 5; done

{"level":"info","ts":1644085993.3831933,"logger":"controller.lvmvolumegroup.vg-manager","msg":"reconciling","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","lvmvolumegroup":"lvm-operator-system/vg3"}
{"level":"info","ts":1644085993.3832853,"logger":"controller.lvmvolumegroup.vg-manager","msg":"listing block devices","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","VGName":"vg3"}
{"level":"info","ts":1644085993.4152625,"logger":"controller.lvmvolumegroup.vg-manager","msg":"does not match filter","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","Device.Name":"sdb","filter.Name":"noChildren"}
{"level":"info","ts":1644085993.4331303,"logger":"controller.lvmvolumegroup.vg-manager","msg":"lvmd config file doesn't exist, will create","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system"}
{"level":"info","ts":1644085993.5088563,"logger":"controller.lvmvolumegroup.vg-manager","msg":"creating a new volume group","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","Name":"vg3"}
{"level":"error","ts":1644085993.6091464,"logger":"controller.lvmvolumegroup.vg-manager","msg":"failed to create/extend volume group","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","VGName":"vg3","error":"failed to create or extend volume group \"vg3\". exit status 5","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":1644085993.6093926,"logger":"controller.lvmvolumegroup.vg-manager","msg":"lvmvolumegroupnodestatus unchanged","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system"}
{"level":"error","ts":1644085993.6094112,"logger":"controller.lvmvolumegroup.vg-manager","msg":"reconcile error","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","error":"failed to create or extend volume group \"vg3\". exit status 5","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":1644085993.6094544,"logger":"controller.lvmvolumegroup.vg-manager","msg":"reconcile complete","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","result":{"Requeue":true,"RequeueAfter":60000000000}}
{"level":"error","ts":1644085993.6095316,"logger":"controller.lvmvolumegroup","msg":"Reconciler error","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg3","namespace":"lvm-operator-system","error":"failed to create or extend volume group \"vg3\". exit status 5","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"}

{"level":"error","ts":1640139614.0910594,"logger":"controller.lvmcluster.vg-manager","msg":"could not prepare volume group","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMCluster","name":"lvmcluster-sample","namespace":"lvm-operator-system","name":"vg1","error":"exit status 5","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"}
{"level":"info","ts":1640139614.091191,"logger":"controller.lvmcluster.vg-manager","msg":"reconcile complete","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMCluster","name":"lvmcluster-sample","namespace":"lvm-operator-system","result":{"Requeue":false,"RequeueAfter":120000000000}}

{"level":"info","ts":1644897063.4425514,"logger":"controller.lvmvolumegroup.vg-manager","msg":"does not match filter","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg1","namespace":"lvm-operator-system","Device.Name":"sdc","filter.Name":"notRemovable"}
{"level":"info","ts":1644897063.449244,"logger":"controller.lvmvolumegroup.vg-manager","msg":"does not match filter","reconciler group":"lvm.topolvm.io","reconciler kind":"LVMVolumeGroup","name":"vg1","namespace":"lvm-operator-system","Device.Name":"sdd","filter.Name":"notRemovable"}```

$> oc get pods
NAME                                  READY   STATUS    RESTARTS   AGE
controller-manager-66b84d759f-9zpv9   3/3     Running   0          2m54s

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: topolvm-provisioner
provisioner: topolvm.cybozu.com
parameters:
  "csi.storage.k8s.io/fstype": "xfs"
  "topolvm.cybozu.com/device-class": "ssd"
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true

$ oc get sc
NAME                            PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
local-sc                        kubernetes.io/no-provisioner   Delete          WaitForFirstConsumer   false                  2d4h
topolvm-provisioner (default)   topolvm.cybozu.com             Delete          WaitForFirstConsumer   true                   8m46s

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: topolvm-pv-claim
spec:
  storageClassName: topolvm-provisioner
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 3Gi

$ oc get pvc
NAME               STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS          AGE
topolvm-pv-claim   Pending                                      topolvm-provisioner   2m43s
$> oc describe pvc topolvm-pv-claim
Name:          topolvm-pv-claim
Namespace:     lvm-operator-system
StorageClass:  topolvm-provisioner
Status:        Pending
Volume:        
Labels:        <none>
Annotations:   volume.beta.kubernetes.io/storage-provisioner: topolvm.cybozu.com
               volume.kubernetes.io/selected-node: master-0.apollo2.hpecloud.org
               volume.kubernetes.io/storage-provisioner: topolvm.cybozu.com
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      
Access Modes:  
VolumeMode:    Filesystem
Used By:       task-pv-pod
Events:
  Type    Reason                Age                   From                         Message
  ----    ------                ----                  ----                         -------
  Normal  WaitForFirstConsumer  3m9s (x2 over 3m19s)  persistentvolume-controller  waiting for first consumer to be created before binding
  Normal  ExternalProvisioning  9s (x14 over 3m6s)    persistentvolume-controller  waiting for a volume to be created, either by external provisioner "topolvm.cybozu.com" or manually created by system administrator

[root@cnf10-worker-0 ~]# oc get pod -n odf-lvm
NAME                                  READY   STATUS             RESTARTS        AGE
controller-manager-765f44745b-hgcpn   3/3     Running            0               31m
topolvm-controller-5ffdc8cd9f-sktg9   4/4     Running            8 (7m41s ago)   31m
topolvm-node-8ffm7                    0/4     Init:0/1           0               31m
topolvm-node-w2rvd                    0/4     Pending            0               31m
topolvm-node-w5s7w                    0/4     Init:0/1           0               31m
vg-manager-8mvn8                      0/1     CrashLoopBackOff   7 (4m4s ago)    31m
vg-manager-mg9xj                      0/1     CrashLoopBackOff   7 (4m19s ago)   31m
vg-manager-wvhbv                      0/1     CrashLoopBackOff   7 (3m53s ago)   31m

[root@cnf10-worker-0 ~]# oc describe pod  -n odf-lvm topolvm-node-8ffm7
Name:         topolvm-node-8ffm7
Namespace:    odf-lvm
Priority:     0
Node:         ci-ovirt-master-0.karmalabs.com/10.19.135.249
Start Time:   Wed, 20 Apr 2022 13:53:03 -0400
Labels:       app=topolvm-node
              controller-revision-hash=5685697cf9
              pod-template-generation=1
Annotations:  k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "openshift-sdn",
                    "interface": "eth0",
                    "ips": [
                        "10.133.0.246"
                    ],
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "openshift-sdn",
                    "interface": "eth0",
                    "ips": [
                        "10.133.0.246"
                    ],
                    "default": true,
                    "dns": {}
                }]
              openshift.io/scc: odf-lvm-topolvm-node
Status:       Pending
IP:           10.133.0.246
IPs:
  IP:           10.133.0.246
Controlled By:  DaemonSet/topolvm-node
Init Containers:
  file-checker:
    Container ID:  cri-o://a8a17b40bc03851f13063e7bb245e4a0214b39411a54ab1ebfabec0b634ef14b
    Image:         registry.redhat.io/odf4/odf-lvm-rhel8-operator@sha256:2bad9a3ab52faf43f8f5258c64ea6734ab40114addfdde116c0bd27d9088bf49
    Image ID:      registry.redhat.io/odf4/odf-lvm-rhel8-operator@sha256:2bad9a3ab52faf43f8f5258c64ea6734ab40114addfdde116c0bd27d9088bf49
    Port:          <none>
    Host Port:     <none>
    Command:
      /usr/bin/bash
      -c
      until [ -f /etc/topolvm/lvmd.yaml ]; do echo waiting for lvmd config file; sleep 5; done
    State:          Running
      Started:      Wed, 20 Apr 2022 13:53:14 -0400
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /etc/topolvm from lvmd-config-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pck6f (ro)
Containers:
  lvmd:
    Container ID:
    Image:         registry.redhat.io/odf4/odf-topolvm-rhel8@sha256:4fb7b673d4a14021df0ad89cd99eed68dd837163bfc32aa8dc8b3eb10d60acee
    Image ID:
    Port:          <none>
    Host Port:     <none>
    Command:
      /lvmd
      --config=/etc/topolvm/lvmd.yaml
      --container=true
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     250m
      memory:  250Mi
    Requests:
      cpu:        250m
      memory:     250Mi
    Environment:  <none>
    Mounts:
      /etc/topolvm from lvmd-config-dir (rw)
      /run/lvmd from lvmd-socket-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pck6f (ro)
  topolvm-node:
    Container ID:
    Image:         registry.redhat.io/odf4/odf-topolvm-rhel8@sha256:4fb7b673d4a14021df0ad89cd99eed68dd837163bfc32aa8dc8b3eb10d60acee
    Image ID:
    Port:          9808/TCP
    Host Port:     0/TCP
    Command:
      /topolvm-node
      --lvmd-socket=/run/lvmd/lvmd.sock
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     250m
      memory:  250Mi
    Requests:
      cpu:     250m
      memory:  250Mi
    Liveness:  http-get http://:healthz/healthz delay=10s timeout=3s period=60s #success=1 #failure=3
    Environment:
      NODE_NAME:   (v1:spec.nodeName)
    Mounts:
      /run/lvmd from lvmd-socket-dir (rw)
      /run/topolvm from node-plugin-dir (rw)
      /var/lib/kubelet/plugins/kubernetes.io/csi from csi-plugin-dir (rw)
      /var/lib/kubelet/pods from pod-volumes-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pck6f (ro)
  csi-registrar:
    Container ID:
    Image:         registry.redhat.io/openshift4/ose-csi-node-driver-registrar@sha256:3308ef98afab494b80aa1a702924407cf114bce6e0ad92436e508d7dc951521c
    Image ID:
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/run/topolvm/csi-topolvm.sock
      --kubelet-registration-path=/var/lib/kubelet/plugins/topolvm.cybozu.com/node/csi-topolvm.sock
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /registration from registration-dir (rw)
      /run/topolvm from node-plugin-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pck6f (ro)
  liveness-probe:
    Container ID:
    Image:         registry.redhat.io/openshift4/ose-csi-livenessprobe@sha256:6b40bb1cb5bffc8e8689b8d01e43096a2d57981aa20ae7859618054ed3800bd7
    Image ID:
    Port:          <none>
    Host Port:     <none>
    Args:
      --csi-address=/run/topolvm/csi-topolvm.sock
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /run/topolvm from node-plugin-dir (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pck6f (ro)
Conditions:
  Type              Status
  Initialized       False
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  registration-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins_registry/
    HostPathType:  Directory
  node-plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/topolvm.cybozu.com/node
    HostPathType:  DirectoryOrCreate
  csi-plugin-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/plugins/kubernetes.io/csi
    HostPathType:  DirectoryOrCreate
  pod-volumes-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/kubelet/pods/
    HostPathType:  DirectoryOrCreate
  lvmd-config-dir:
    Type:          HostPath (bare host directory volume)
    Path:          /etc/topolvm
    HostPathType:  Directory
  lvmd-socket-dir:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  kube-api-access-pck6f:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    ConfigMapOptional:       <nil>
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/disk-pressure:NoSchedule op=Exists
                             node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists
                             node.kubernetes.io/pid-pressure:NoSchedule op=Exists
                             node.kubernetes.io/unreachable:NoExecute op=Exists
                             node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
  Type    Reason          Age   From               Message
  ----    ------          ----  ----               -------
  Normal  Scheduled       31m   default-scheduler  Successfully assigned odf-lvm/topolvm-node-8ffm7 to ci-ovirt-master-0.karmalabs.com
  Normal  AddedInterface  31m   multus             Add eth0 [10.133.0.246/23] from openshift-sdn
  Normal  Pulling         31m   kubelet            Pulling image "registry.redhat.io/odf4/odf-lvm-rhel8-operator@sha256:2bad9a3ab52faf43f8f5258c64ea6734ab40114addfdde116c0bd27d9088bf49"
  Normal  Pulled          30m   kubelet            Successfully pulled image "registry.redhat.io/odf4/odf-lvm-rhel8-operator@sha256:2bad9a3ab52faf43f8f5258c64ea6734ab40114addfdde116c0bd27d9088bf49" in 7.936196491s
  Normal  Created         30m   kubelet            Created container file-checker
  Normal  Started         30m   kubelet            Started container file-checker

Failed to pull image "alpine/openssl": rpc error: code = Unknown desc = initializing source docker://alpine/openssl:latest: reading manifest latest in docker.io/alpine/openssl: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit