root@sbat1:~# kubectl get all --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES default pod/onos-6f54986475-rrhj5 2/2 Running 0 2d17h 10.233.79.32 sbat3 kube-system pod/calico-kube-controllers-d9587b546-7t95q 1/1 Running 4 20d xxx.xx.xx.167 sbat1 kube-system pod/calico-node-4r7rf 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/calico-node-p5sfv 1/1 Running 4 20d xxx.xx.xx.167 sbat1 kube-system pod/calico-node-prsrt 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/coredns-56bc6b976d-522sf 1/1 Running 0 20d 10.233.79.2 sbat3 kube-system pod/coredns-56bc6b976d-gvpwn 1/1 Running 0 20d 10.233.83.1 sbat2 kube-system pod/dns-autoscaler-56c969bdb8-pwshs 1/1 Running 4 20d 10.233.70.32 sbat1 kube-system pod/kube-apiserver-sbat1 1/1 Running 4 20d xxx.xx.xx.167 sbat1 kube-system pod/kube-apiserver-sbat2 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/kube-apiserver-sbat3 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/kube-controller-manager-sbat1 1/1 Running 5 20d xxx.xx.xx.167 sbat1 kube-system pod/kube-controller-manager-sbat2 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/kube-controller-manager-sbat3 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/kube-proxy-5rz7j 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/kube-proxy-8kg7f 1/1 Running 4 20d xxx.xx.xx.167 sbat1 kube-system pod/kube-proxy-qjrww 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/kube-scheduler-sbat1 1/1 Running 5 20d xxx.xx.xx.167 sbat1 kube-system pod/kube-scheduler-sbat2 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/kube-scheduler-sbat3 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/kubernetes-dashboard-6c7466966c-2lttd 1/1 Running 5 20d 10.233.70.31 sbat1 kube-system pod/nodelocaldns-hz5bf 1/1 Running 4 20d xxx.xx.xx.167 sbat1 kube-system pod/nodelocaldns-qbvxj 1/1 Running 0 20d xxx.xx.xx.157 sbat2 kube-system pod/nodelocaldns-znxtl 1/1 Running 0 20d xxx.xx.xx.151 sbat3 kube-system pod/tiller-deploy-86c8b7897c-sj6ch 1/1 Running 0 20d 10.233.83.2 sbat2 voltha pod/adapter-open-olt-8cf67d66b-msngz 1/1 Running 0 2d17h 10.233.70.46 sbat1 voltha pod/adapter-open-onu-7b86fc69f6-xgrm8 1/1 Running 0 2d17h 10.233.70.47 sbat1 voltha pod/ofagent-96d8bdd8-qlh7g 1/1 Running 1 2d17h 10.233.83.35 sbat2 voltha pod/ro-core-6d7f69b84-sdkmh 1/1 Running 0 2d17h 10.233.79.34 sbat3 voltha pod/rw-core1-69fccb85c5-txf7l 1/1 Running 3 2d17h 10.233.79.35 sbat3 voltha pod/rw-core2-5c7475dfc5-zhm92 1/1 Running 3 2d17h 10.233.70.43 sbat1 voltha pod/voltha-api-server-dbfbcb66b-qvrlc 2/2 Running 0 2d17h 10.233.83.33 sbat2 voltha pod/voltha-cli-server-688c9599c6-vtrng 1/1 Running 0 2d17h 10.233.79.33 sbat3 voltha pod/voltha-etcd-cluster-86qr8rqjd5 0/1 OOMKilled 0 2d17h 10.233.70.45 sbat1 voltha pod/voltha-etcd-cluster-bsvjlddgmf 0/1 Completed 0 2d17h sbat2 voltha pod/voltha-etcd-cluster-dr8ksmw4x8 0/1 OOMKilled 0 2d17h 10.233.79.37 sbat3 voltha pod/voltha-etcd-operator-etcd-backup-operator-b9bc4b645-8c9mp 1/1 Running 0 2d17h 10.233.83.36 sbat2 voltha pod/voltha-etcd-operator-etcd-operator-774dbc6954-2rd4n 1/1 Running 0 2d17h 10.233.83.37 sbat2 voltha pod/voltha-etcd-operator-etcd-restore-operator-6777b84b45-qbbwr 1/1 Running 0 2d17h 10.233.70.44 sbat1 voltha pod/voltha-kafka-0 1/1 Running 0 2d17h 10.233.79.36 sbat3 voltha pod/voltha-zookeeper-0 1/1 Running 0 2d17h 10.233.83.34 sbat2 NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR default service/kubernetes ClusterIP 10.233.0.1 443/TCP 20d default service/onos-debugger NodePort 10.233.63.250 5005:30555/TCP 2d17h app=onos,release=onos default service/onos-openflow NodePort 10.233.4.61 6653:31653/TCP 2d17h app=onos,release=onos default service/onos-ovsdb ClusterIP 10.233.37.28 6640/TCP 2d17h app=onos,release=onos default service/onos-ssh NodePort 10.233.21.174 8101:30115/TCP 2d17h app=onos,release=onos default service/onos-ui NodePort 10.233.60.79 8181:30120/TCP 2d17h app=onos,release=onos kube-system service/coredns ClusterIP 10.233.0.3 53/UDP,53/TCP,9153/TCP 20d k8s-app=kube-dns kube-system service/kubernetes-dashboard ClusterIP 10.233.24.91 443/TCP 20d k8s-app=kubernetes-dashboard kube-system service/tiller-deploy ClusterIP 10.233.63.65 44134/TCP 20d app=helm,name=tiller voltha service/etcd-restore-operator ClusterIP 10.233.56.34 19999/TCP 2d17h app=etcd-restore-operator,release=voltha voltha service/voltha-api ClusterIP 10.233.58.105 55555/TCP 2d17h app=voltha-api-server voltha service/voltha-cli ClusterIP 10.233.20.101 5022/TCP 2d17h app=vcli voltha service/voltha-etcd-cluster ClusterIP None 2379/TCP,2380/TCP 2d17h app=etcd,etcd_cluster=voltha-etcd-cluster voltha service/voltha-etcd-cluster-client ClusterIP 10.233.17.31 2379/TCP 2d17h app=etcd,etcd_cluster=voltha-etcd-cluster voltha service/voltha-kafka ClusterIP 10.233.47.78 9092/TCP 2d17h app=kafka,release=voltha voltha service/voltha-kafka-headless ClusterIP None 9092/TCP 2d17h app=kafka,release=voltha voltha service/voltha-zookeeper ClusterIP 10.233.3.236 2181/TCP 2d17h app=zookeeper,release=voltha voltha service/voltha-zookeeper-headless ClusterIP None 2181/TCP,3888/TCP,2888/TCP 2d17h app=zookeeper,release=voltha NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE CONTAINERS IMAGES SELECTOR kube-system daemonset.apps/calico-node 3 3 3 3 3 20d calico-node docker.io/calico/node:v3.4.0 k8s-app=calico-node kube-system daemonset.apps/kube-proxy 3 3 3 3 3 beta.kubernetes.io/os=linux 20d kube-proxy gcr.io/google-containers/kube-proxy:v1.14.1 k8s-app=kube-proxy kube-system daemonset.apps/nodelocaldns 3 3 3 3 3 20d node-cache k8s.gcr.io/k8s-dns-node-cache:1.15.1 k8s-app=nodelocaldns NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR default deployment.apps/onos 1/1 1 1 2d17h onos,onos-log-agent onosproject/onos:1.13.5,docker.elastic.co/beats/filebeat-oss:6.4.2 app=onos,release=onos kube-system deployment.apps/calico-kube-controllers 1/1 1 1 20d calico-kube-controllers docker.io/calico/kube-controllers:v3.4.0 k8s-app=calico-kube-controllers,kubernetes.io/cluster-service=true kube-system deployment.apps/coredns 2/2 2 2 20d coredns docker.io/coredns/coredns:1.5.0 k8s-app=kube-dns kube-system deployment.apps/dns-autoscaler 1/1 1 1 20d autoscaler k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.4.0 k8s-app=dns-autoscaler kube-system deployment.apps/kubernetes-dashboard 1/1 1 1 20d kubernetes-dashboard gcr.io/google_containers/kubernetes-dashboard-amd64:v1.10.1 k8s-app=kubernetes-dashboard kube-system deployment.apps/tiller-deploy 1/1 1 1 20d tiller gcr.io/kubernetes-helm/tiller:v2.13.1 app=helm,name=tiller voltha deployment.apps/adapter-open-olt 1/1 1 1 2d17h adapter-open-olt voltha/voltha-openolt-adapter:2.0.0 app=adapter-open-olt voltha deployment.apps/adapter-open-onu 1/1 1 1 2d17h adapter-open-onu voltha/voltha-openonu-adapter:2.0.0 app=adapter-open-onu voltha deployment.apps/ofagent 1/1 1 1 2d17h ofagent volthacore/voltha-ofagent:2.0.0 app=ofagent voltha deployment.apps/ro-core 1/1 1 1 2d17h voltha volthacore/voltha-ro-core:2.0.0 app=ro-core voltha deployment.apps/rw-core1 1/1 1 1 2d17h voltha volthacore/voltha-rw-core:2.0.0 app=rw-core voltha deployment.apps/rw-core2 1/1 1 1 2d17h voltha volthacore/voltha-rw-core:2.0.0 app=rw-core voltha deployment.apps/voltha-api-server 1/1 1 1 2d17h arouter,arouterd volthacore/voltha-afrouter:2.0.0,volthacore/voltha-afrouterd:2.0.0 app=voltha-api-server voltha deployment.apps/voltha-cli-server 1/1 1 1 2d17h vcli volthacore/voltha-cli:2.0.0 app=vcli voltha deployment.apps/voltha-etcd-operator-etcd-backup-operator 1/1 1 1 2d17h etcd-backup-operator quay.io/coreos/etcd-operator:v0.9.3 app=voltha-etcd-operator-etcd-backup-operator,release=voltha voltha deployment.apps/voltha-etcd-operator-etcd-operator 1/1 1 1 2d17h voltha-etcd-operator-etcd-operator quay.io/coreos/etcd-operator:v0.9.3 app=voltha-etcd-operator-etcd-operator,release=voltha voltha deployment.apps/voltha-etcd-operator-etcd-restore-operator 1/1 1 1 2d17h etcd-restore-operator quay.io/coreos/etcd-operator:v0.9.3 app=etcd-restore-operator,release=voltha NAMESPACE NAME DESIRED CURRENT READY AGE CONTAINERS IMAGES SELECTOR default replicaset.apps/onos-6f54986475 1 1 1 2d17h onos,onos-log-agent onosproject/onos:1.13.5,docker.elastic.co/beats/filebeat-oss:6.4.2 app=onos,pod-template-hash=6f54986475,release=onos kube-system replicaset.apps/calico-kube-controllers-d9587b546 1 1 1 20d calico-kube-controllers docker.io/calico/kube-controllers:v3.4.0 k8s-app=calico-kube-controllers,kubernetes.io/cluster-service=true,pod-template-hash=d9587b546 kube-system replicaset.apps/coredns-56bc6b976d 2 2 2 20d coredns docker.io/coredns/coredns:1.5.0 k8s-app=kube-dns,pod-template-hash=56bc6b976d kube-system replicaset.apps/dns-autoscaler-56c969bdb8 1 1 1 20d autoscaler k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.4.0 k8s-app=dns-autoscaler,pod-template-hash=56c969bdb8 kube-system replicaset.apps/kubernetes-dashboard-6c7466966c 1 1 1 20d kubernetes-dashboard gcr.io/google_containers/kubernetes-dashboard-amd64:v1.10.1 k8s-app=kubernetes-dashboard,pod-template-hash=6c7466966c kube-system replicaset.apps/tiller-deploy-86c8b7897c 1 1 1 20d tiller gcr.io/kubernetes-helm/tiller:v2.13.1 app=helm,name=tiller,pod-template-hash=86c8b7897c voltha replicaset.apps/adapter-open-olt-8cf67d66b 1 1 1 2d17h adapter-open-olt voltha/voltha-openolt-adapter:2.0.0 app=adapter-open-olt,pod-template-hash=8cf67d66b voltha replicaset.apps/adapter-open-onu-7b86fc69f6 1 1 1 2d17h adapter-open-onu voltha/voltha-openonu-adapter:2.0.0 app=adapter-open-onu,pod-template-hash=7b86fc69f6 voltha replicaset.apps/ofagent-96d8bdd8 1 1 1 2d17h ofagent volthacore/voltha-ofagent:2.0.0 app=ofagent,pod-template-hash=96d8bdd8 voltha replicaset.apps/ro-core-6d7f69b84 1 1 1 2d17h voltha volthacore/voltha-ro-core:2.0.0 app=ro-core,pod-template-hash=6d7f69b84 voltha replicaset.apps/rw-core1-69fccb85c5 1 1 1 2d17h voltha volthacore/voltha-rw-core:2.0.0 app=rw-core,pod-template-hash=69fccb85c5 voltha replicaset.apps/rw-core2-5c7475dfc5 1 1 1 2d17h voltha volthacore/voltha-rw-core:2.0.0 app=rw-core,pod-template-hash=5c7475dfc5 voltha replicaset.apps/voltha-api-server-dbfbcb66b 1 1 1 2d17h arouter,arouterd volthacore/voltha-afrouter:2.0.0,volthacore/voltha-afrouterd:2.0.0 app=voltha-api-server,pod-template-hash=dbfbcb66b voltha replicaset.apps/voltha-cli-server-688c9599c6 1 1 1 2d17h vcli volthacore/voltha-cli:2.0.0 app=vcli,pod-template-hash=688c9599c6 voltha replicaset.apps/voltha-etcd-operator-etcd-backup-operator-b9bc4b645 1 1 1 2d17h etcd-backup-operator quay.io/coreos/etcd-operator:v0.9.3 app=voltha-etcd-operator-etcd-backup-operator,pod-template-hash=b9bc4b645,release=voltha voltha replicaset.apps/voltha-etcd-operator-etcd-operator-774dbc6954 1 1 1 2d17h voltha-etcd-operator-etcd-operator quay.io/coreos/etcd-operator:v0.9.3 app=voltha-etcd-operator-etcd-operator,pod-template-hash=774dbc6954,release=voltha voltha replicaset.apps/voltha-etcd-operator-etcd-restore-operator-6777b84b45 1 1 1 2d17h etcd-restore-operator quay.io/coreos/etcd-operator:v0.9.3 app=etcd-restore-operator,pod-template-hash=6777b84b45,release=voltha NAMESPACE NAME READY AGE CONTAINERS IMAGES voltha statefulset.apps/voltha-kafka 1/1 2d17h kafka-broker confluentinc/cp-kafka:5.0.1 voltha statefulset.apps/voltha-zookeeper 1/1 2d17h zookeeper gcr.io/google_samples/k8szk:v3 root@sbat1:~# kubectl get pod voltha-etcd-cluster-dr8ksmw4x8 --output=yaml --namespace voltha apiVersion: v1 kind: Pod metadata: annotations: etcd.version: 3.2.18 creationTimestamp: "2019-06-07T18:32:12Z" labels: app: etcd etcd_cluster: voltha-etcd-cluster etcd_node: voltha-etcd-cluster-dr8ksmw4x8 name: voltha-etcd-cluster-dr8ksmw4x8 namespace: voltha ownerReferences: - apiVersion: etcd.database.coreos.com/v1beta2 controller: true kind: EtcdCluster name: voltha-etcd-cluster uid: 5e4b1572-8952-11e9-8921-288023a0c20c resourceVersion: "3309764" selfLink: /api/v1/namespaces/voltha/pods/voltha-etcd-cluster-dr8ksmw4x8 uid: 9134bb06-8952-11e9-af07-288023a0afa0 spec: automountServiceAccountToken: false containers: - command: - /usr/local/bin/etcd - --data-dir=/var/etcd/data - --name=voltha-etcd-cluster-dr8ksmw4x8 - --initial-advertise-peer-urls=http://voltha-etcd-cluster-dr8ksmw4x8.voltha-etcd-cluster.voltha.svc:2380 - --listen-peer-urls=http://0.0.0.0:2380 - --listen-client-urls=http://0.0.0.0:2379 - --advertise-client-urls=http://voltha-etcd-cluster-dr8ksmw4x8.voltha-etcd-cluster.voltha.svc:2379 - --initial-cluster=voltha-etcd-cluster-86qr8rqjd5=http://voltha-etcd-cluster-86qr8rqjd5.voltha-etcd-cluster.voltha.svc:2380,voltha-etcd-cluster-dr8ksmw4x8=http://voltha-etcd-cluster-dr8ksmw4x8.voltha-etcd-cluster.voltha.svc:2380 - --initial-cluster-state=existing env: - name: ETCD_AUTO_COMPACTION_RETENTION value: "12" image: quay.io/coreos/etcd:v3.2.18 imagePullPolicy: IfNotPresent livenessProbe: exec: command: - /bin/sh - -ec - ETCDCTL_API=3 etcdctl endpoint health failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 60 successThreshold: 1 timeoutSeconds: 10 name: etcd ports: - containerPort: 2380 name: server protocol: TCP - containerPort: 2379 name: client protocol: TCP readinessProbe: exec: command: - /bin/sh - -ec - ETCDCTL_API=3 etcdctl endpoint health failureThreshold: 3 initialDelaySeconds: 1 periodSeconds: 5 successThreshold: 1 timeoutSeconds: 5 resources: limits: memory: 1Gi requests: memory: 1Gi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/etcd name: etcd-data dnsPolicy: ClusterFirst enableServiceLinks: true hostname: voltha-etcd-cluster-dr8ksmw4x8 initContainers: - command: - /bin/sh - -c - "\n\t\t\t\t\tTIMEOUT_READY=0\n\t\t\t\t\twhile ( ! nslookup voltha-etcd-cluster-dr8ksmw4x8.voltha-etcd-cluster.voltha.svc )\n\t\t\t\t\tdo\n\t\t\t\t\t\t# If TIMEOUT_READY is 0 we should never time out and exit \n\t\t\t\t\t\tTIMEOUT_READY=$(( TIMEOUT_READY-1 ))\n if [ $TIMEOUT_READY -eq 0 ];\n\t\t\t\t then\n\t\t\t\t echo \"Timed out waiting for DNS entry\"\n\t\t\t\t exit 1\n\t\t\t\t fi\n\t\t\t\t\t\tsleep 1\n\t\t\t\t\tdone" image: busybox:1.28.1-glibc imagePullPolicy: IfNotPresent name: check-dns resources: limits: memory: 1Gi requests: memory: 1Gi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File nodeName: sbat3 priority: 0 restartPolicy: Never schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default subdomain: voltha-etcd-cluster terminationGracePeriodSeconds: 30 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 volumes: - emptyDir: {} name: etcd-data status: conditions: - lastProbeTime: null lastTransitionTime: "2019-06-07T18:32:21Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2019-06-08T03:17:13Z" message: 'containers with unready status: [etcd]' reason: ContainersNotReady status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2019-06-08T03:17:13Z" message: 'containers with unready status: [etcd]' reason: ContainersNotReady status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2019-06-07T18:31:29Z" status: "True" type: PodScheduled containerStatuses: - containerID: docker://884b7b42e19c02a48fd9ffea2df9db4fafd014c987dd0272a6518d9d1a6a5dea image: quay.io/coreos/etcd:v3.2.18 imageID: docker-pullable://quay.io/coreos/etcd@sha256:1f433f387d7d0ff283e9abfdd493a461175bee0c372deca68c5633eb7ce7ee56 lastState: {} name: etcd ready: false restartCount: 0 state: terminated: containerID: docker://884b7b42e19c02a48fd9ffea2df9db4fafd014c987dd0272a6518d9d1a6a5dea exitCode: 137 finishedAt: "2019-06-08T03:17:12Z" reason: OOMKilled startedAt: "2019-06-07T18:32:21Z" hostIP: xxx.xx.xx.151 initContainerStatuses: - containerID: docker://5312ea64ea85d6551d39c201421285e49c145bce246528ded4e8d3e51abddabe image: busybox:1.28.1-glibc imageID: docker-pullable://busybox@sha256:32bb380c15d34cbdd90e74ceb90bbbcb6d5567fcf3d706a9338617d5a878dcdb lastState: {} name: check-dns ready: true restartCount: 0 state: terminated: containerID: docker://5312ea64ea85d6551d39c201421285e49c145bce246528ded4e8d3e51abddabe exitCode: 0 finishedAt: "2019-06-07T18:32:20Z" reason: Completed startedAt: "2019-06-07T18:32:15Z" phase: Failed podIP: 10.233.79.37 qosClass: Burstable startTime: "2019-06-07T18:32:12Z"