-
Type: Bug
-
Status: To Do (View Workflow)
-
Priority: Medium
-
Resolution: Unresolved
-
Affects Version/s: SEBA 2.0
-
Fix Version/s: Future
-
Component/s: Kubernetes
-
Labels:None
-
Environment:
root@sbat1:~# kubectl describe pods --all-namespaces | grep Image:
Image: xosproject/att-workflow-driver-synchronizer:1.2.3
Image: xosproject/tosca-loader:1.3.1
Image: busybox:latest
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: busybox:latest
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: busybox:latest
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: busybox:latest
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4
Image: quay.io/coreos/etcd-operator:v0.9.3
Image: quay.io/coreos/etcd-operator:v0.9.3
Image: quay.io/coreos/etcd-operator:v0.9.3
Image: kiwigrid/k8s-sidecar:0.0.3
Image: grafana/grafana:5.1.3
Image: confluentinc/cp-kafka:5.0.1
Image: docker.elastic.co/kibana/kibana-oss:6.5.4
Image: docker.elastic.co/kibana/kibana-oss:6.5.4
Image: docker.elastic.co/logstash/logstash-oss:6.5.4
Image: onosproject/onos:1.13.9
Image: docker.elastic.co/beats/filebeat-oss:6.4.2
Image: prom/alertmanager:v0.15.0
Image: jimmidyson/configmap-reload:v0.2.2
Image: quay.io/coreos/kube-state-metrics:v1.3.1
Image: prom/node-exporter:v0.16.0
Image: prom/node-exporter:v0.16.0
Image: prom/node-exporter:v0.16.0
Image: prom/pushgateway:v0.5.2
Image: busybox:latest
Image: jimmidyson/configmap-reload:v0.2.2
Image: prom/prometheus:v2.3.1
Image: gcr.io/google_samples/k8szk:v3
Image: busybox:1.28.1-glibc
Image: quay.io/coreos/etcd:v3.2.18
Image: opencord/kafka-topic-exporter:1.2.3
Image: opencord/kafka-topic-exporter:1.2.3
Image: opencord/sadis-server:1.1.0
Image: xosproject/tosca-loader:1.3.1
Image: xosproject/fabric-synchronizer:2.2.2
Image: xosproject/fabric-crossconnect-synchronizer:1.2.2
Image: xosproject/kubernetes-synchronizer:1.3.1
Image: xosproject/onos-synchronizer:2.1.2
Image: xosproject/rcord-synchronizer:1.3.2
Image: xosproject/tosca-loader:1.3.1
Image: xosproject/volt-synchronizer:2.2.4
Image: xosproject/chameleon:3.3.0
Image: xosproject/xos-core:3.3.1
Image: postgres:10.3-alpine
Image: xosproject/xos-gui:1.0.6
Image: xosproject/xos-tosca:1.3.0
Image: xosproject/xos-rest-gw:2.0.2
Image: docker.io/calico/kube-controllers:v3.4.0
Image: docker.io/calico/cni:v3.4.0
Image: docker.io/calico/node:v3.4.0
Image: docker.io/calico/cni:v3.4.0
Image: docker.io/calico/node:v3.4.0
Image: docker.io/calico/cni:v3.4.0
Image: docker.io/calico/node:v3.4.0
Image: docker.io/coredns/coredns:1.5.0
Image: docker.io/coredns/coredns:1.5.0
Image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.4.0
Image: gcr.io/google-containers/kube-apiserver:v1.14.3
Image: gcr.io/google-containers/kube-apiserver:v1.14.3
Image: gcr.io/google-containers/kube-apiserver:v1.14.3
Image: gcr.io/google-containers/kube-controller-manager:v1.14.3
Image: gcr.io/google-containers/kube-controller-manager:v1.14.3
Image: gcr.io/google-containers/kube-controller-manager:v1.14.3
Image: gcr.io/google-containers/kube-proxy:v1.14.3
Image: gcr.io/google-containers/kube-proxy:v1.14.3
Image: gcr.io/google-containers/kube-proxy:v1.14.3
Image: gcr.io/google-containers/kube-scheduler:v1.14.3
Image: gcr.io/google-containers/kube-scheduler:v1.14.3
Image: gcr.io/google-containers/kube-scheduler:v1.14.3
Image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.10.1
Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1
Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1
Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1
Image: gcr.io/kubernetes-helm/tiller:v2.13.1
Image: gcr.io/google_containers/defaultbackend:1.4
Image: tpdock/freeradius:2.2.9
Image: voltha/voltha-netconf:1.7.0
Image: alpine:3.6
Image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.10.2
Image: voltha/voltha-ofagent:1.7.0
Image: voltha/voltha-cli:1.7.0
Image: voltha/voltha-voltha:1.7.0
Image: voltha/voltha-envoy:1.7.0root@sbat1:~# kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
default att-workflow-att-workflow-driver-f688bb799-xb2lc 1/1 Running 0 3d23h
default att-workflow-tosca-loader-r5hns 0/1 Completed 0 3d23h
default cord-platform-elasticsearch-client-64764f9887-24dk2 1/1 Running 0 3d23h
default cord-platform-elasticsearch-data-0 1/1 Running 0 3d23h
default cord-platform-elasticsearch-master-0 1/1 Running 0 3d23h
default cord-platform-elasticsearch-master-1 1/1 Running 0 3d23h
default cord-platform-etcd-operator-etcd-backup-operator-64c545cf6dpk8v 1/1 Running 0 3d23h
default cord-platform-etcd-operator-etcd-operator-759567f574-zrhw7 1/1 Running 0 3d23h
default cord-platform-etcd-operator-etcd-restore-operator-7bf97bd4hvzd9 1/1 Running 0 3d23h
default cord-platform-grafana-85d4d648b9-chx2f 2/2 Running 0 3d23h
default cord-platform-kafka-0 0/1 CrashLoopBackOff 929 3d23h
default cord-platform-kibana-5f5d6f57c4-89dh5 1/1 Running 0 3d23h
default cord-platform-logstash-0 1/1 Running 0 3d23h
default cord-platform-onos-5b4bd8d876-6pfb7 2/2 Running 0 3d23h
default cord-platform-prometheus-alertmanager-54c4d7b66c-lq6q6 2/2 Running 0 3d23h
default cord-platform-prometheus-kube-state-metrics-85649df446-n8psv 1/1 Running 0 3d23h
default cord-platform-prometheus-node-exporter-4fj2j 1/1 Running 0 3d23h
default cord-platform-prometheus-node-exporter-mkjn4 1/1 Running 0 3d23h
default cord-platform-prometheus-node-exporter-nhtpb 1/1 Running 0 3d23h
default cord-platform-prometheus-pushgateway-68cdb5847-rwnjk 1/1 Running 0 3d23h
default cord-platform-prometheus-server-9f9c454fb-bnhmc 2/2 Running 0 3d23h
default cord-platform-zookeeper-0 1/1 Running 0 3d23h
default etcd-cluster-6h2hfdkqvs 1/1 Running 0 4d
default kpi-exporter-85c67bf99-4xrtf 1/1 Running 2 3d23h
default kpi-exporter-85c67bf99-9nzpd 1/1 Running 2 3d23h
default sadis-server-687855dbff-hxzhh 1/1 Running 0 3d23h
default seba-base-kubernetes-tosca-loader-bxbj5 0/1 Completed 0 3d23h
default seba-fabric-59b446c5df-2gwqs 1/1 Running 0 3d23h
default seba-fabric-crossconnect-5c87866689-r44pk 1/1 Running 0 3d23h
default seba-kubernetes-76d9dbf86b-7nwgw 1/1 Running 0 3d23h
default seba-onos-service-649c575954-gv8j5 1/1 Running 1 3d23h
default seba-rcord-5bb8d7bb68-bpg62 1/1 Running 0 3d23h
default seba-seba-services-tosca-loader-lmhvj 0/1 Completed 0 3d23h
default seba-volt-75d9466c4b-6sfwd 1/1 Running 0 3d23h
default xos-chameleon-bc56d9cf9-zrkbx 1/1 Running 0 3d23h
default xos-core-56ff8b89dd-bzjl4 1/1 Running 0 3d23h
default xos-db-76f8db9cd8-t5vqx 1/1 Running 0 3d23h
default xos-gui-6b658dd5df-ln7nr 1/1 Running 0 3d23h
default xos-tosca-5b86cfb5bf-gqxck 1/1 Running 0 3d23h
default xos-ws-d454bd6d4-mssns 1/1 Running 13 3d23h
kube-system calico-kube-controllers-86b89b9949-nxxvt 1/1 Running 3 31d
kube-system calico-node-9xsjt 1/1 Running 2 31d
kube-system calico-node-d2fjb 1/1 Running 2 31d
kube-system calico-node-zk44l 1/1 Running 2 31d
kube-system coredns-97c4b444f-fvgzg 1/1 Running 1 31d
kube-system coredns-97c4b444f-ngxtf 1/1 Running 2 31d
kube-system dns-autoscaler-5fc5fdbf6-c48mk 1/1 Running 2 31d
kube-system kube-apiserver-sbat1 1/1 Running 4 31d
kube-system kube-apiserver-sbat2 1/1 Running 2 31d
kube-system kube-apiserver-sbat3 1/1 Running 2 31d
kube-system kube-controller-manager-sbat1 1/1 Running 5 31d
kube-system kube-controller-manager-sbat2 1/1 Running 2 31d
kube-system kube-controller-manager-sbat3 1/1 Running 2 31d
kube-system kube-proxy-w8rlt 1/1 Running 2 31d
kube-system kube-proxy-wr9gc 1/1 Running 2 31d
kube-system kube-proxy-zcvq4 1/1 Running 3 31d
kube-system kube-scheduler-sbat1 1/1 Running 5 31d
kube-system kube-scheduler-sbat2 1/1 Running 4 31d
kube-system kube-scheduler-sbat3 1/1 Running 3 31d
kube-system kubernetes-dashboard-6c7466966c-2nl88 1/1 Running 2 31d
kube-system nodelocaldns-c7ws4 1/1 Running 5 31d
kube-system nodelocaldns-fgpmc 1/1 Running 2 31d
kube-system nodelocaldns-kcp4h 1/1 Running 2 31d
kube-system tiller-deploy-86c8b7897c-d72kq 1/1 Running 1 31d
voltha default-http-backend-56ddf5b59-h8bgl 1/1 Running 0 3d23h
voltha freeradius-5b469f597f-2852j 1/1 Running 0 3d23h
voltha netconf-54974f6495-5mzsb 1/1 Running 0 3d23h
voltha nginx-ingress-controller-59f977b6b4-tglws 1/1 Running 0 3d23h
voltha ofagent-7b74788989-svbhv 1/1 Running 0 3d23h
voltha vcli-65476cf646-bdqhs 1/1 Running 0 3d23h
voltha vcore-0 1/1 Running 0 3d23h
voltha voltha-5d76b54bcb-lbqfl 1/1 Running 0 3d23hroot@sbat1:~# kubectl describe pods --all-namespaces | grep Image: Image: xosproject/att-workflow-driver-synchronizer:1.2.3 Image: xosproject/tosca-loader:1.3.1 Image: busybox:latest Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: busybox:latest Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: busybox:latest Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: busybox:latest Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.5.4 Image: quay.io/coreos/etcd-operator:v0.9.3 Image: quay.io/coreos/etcd-operator:v0.9.3 Image: quay.io/coreos/etcd-operator:v0.9.3 Image: kiwigrid/k8s-sidecar:0.0.3 Image: grafana/grafana:5.1.3 Image: confluentinc/cp-kafka:5.0.1 Image: docker.elastic.co/kibana/kibana-oss:6.5.4 Image: docker.elastic.co/kibana/kibana-oss:6.5.4 Image: docker.elastic.co/logstash/logstash-oss:6.5.4 Image: onosproject/onos:1.13.9 Image: docker.elastic.co/beats/filebeat-oss:6.4.2 Image: prom/alertmanager:v0.15.0 Image: jimmidyson/configmap-reload:v0.2.2 Image: quay.io/coreos/kube-state-metrics:v1.3.1 Image: prom/node-exporter:v0.16.0 Image: prom/node-exporter:v0.16.0 Image: prom/node-exporter:v0.16.0 Image: prom/pushgateway:v0.5.2 Image: busybox:latest Image: jimmidyson/configmap-reload:v0.2.2 Image: prom/prometheus:v2.3.1 Image: gcr.io/google_samples/k8szk:v3 Image: busybox:1.28.1-glibc Image: quay.io/coreos/etcd:v3.2.18 Image: opencord/kafka-topic-exporter:1.2.3 Image: opencord/kafka-topic-exporter:1.2.3 Image: opencord/sadis-server:1.1.0 Image: xosproject/tosca-loader:1.3.1 Image: xosproject/fabric-synchronizer:2.2.2 Image: xosproject/fabric-crossconnect-synchronizer:1.2.2 Image: xosproject/kubernetes-synchronizer:1.3.1 Image: xosproject/onos-synchronizer:2.1.2 Image: xosproject/rcord-synchronizer:1.3.2 Image: xosproject/tosca-loader:1.3.1 Image: xosproject/volt-synchronizer:2.2.4 Image: xosproject/chameleon:3.3.0 Image: xosproject/xos-core:3.3.1 Image: postgres:10.3-alpine Image: xosproject/xos-gui:1.0.6 Image: xosproject/xos-tosca:1.3.0 Image: xosproject/xos-rest-gw:2.0.2 Image: docker.io/calico/kube-controllers:v3.4.0 Image: docker.io/calico/cni:v3.4.0 Image: docker.io/calico/node:v3.4.0 Image: docker.io/calico/cni:v3.4.0 Image: docker.io/calico/node:v3.4.0 Image: docker.io/calico/cni:v3.4.0 Image: docker.io/calico/node:v3.4.0 Image: docker.io/coredns/coredns:1.5.0 Image: docker.io/coredns/coredns:1.5.0 Image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.4.0 Image: gcr.io/google-containers/kube-apiserver:v1.14.3 Image: gcr.io/google-containers/kube-apiserver:v1.14.3 Image: gcr.io/google-containers/kube-apiserver:v1.14.3 Image: gcr.io/google-containers/kube-controller-manager:v1.14.3 Image: gcr.io/google-containers/kube-controller-manager:v1.14.3 Image: gcr.io/google-containers/kube-controller-manager:v1.14.3 Image: gcr.io/google-containers/kube-proxy:v1.14.3 Image: gcr.io/google-containers/kube-proxy:v1.14.3 Image: gcr.io/google-containers/kube-proxy:v1.14.3 Image: gcr.io/google-containers/kube-scheduler:v1.14.3 Image: gcr.io/google-containers/kube-scheduler:v1.14.3 Image: gcr.io/google-containers/kube-scheduler:v1.14.3 Image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.10.1 Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1 Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1 Image: k8s.gcr.io/k8s-dns-node-cache:1.15.1 Image: gcr.io/kubernetes-helm/tiller:v2.13.1 Image: gcr.io/google_containers/defaultbackend:1.4 Image: tpdock/freeradius:2.2.9 Image: voltha/voltha-netconf:1.7.0 Image: alpine:3.6 Image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.10.2 Image: voltha/voltha-ofagent:1.7.0 Image: voltha/voltha-cli:1.7.0 Image: voltha/voltha-voltha:1.7.0 Image: voltha/voltha-envoy:1.7.0 root@sbat1:~# kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE default att-workflow-att-workflow-driver-f688bb799-xb2lc 1/1 Running 0 3d23h default att-workflow-tosca-loader-r5hns 0/1 Completed 0 3d23h default cord-platform-elasticsearch-client-64764f9887-24dk2 1/1 Running 0 3d23h default cord-platform-elasticsearch-data-0 1/1 Running 0 3d23h default cord-platform-elasticsearch-master-0 1/1 Running 0 3d23h default cord-platform-elasticsearch-master-1 1/1 Running 0 3d23h default cord-platform-etcd-operator-etcd-backup-operator-64c545cf6dpk8v 1/1 Running 0 3d23h default cord-platform-etcd-operator-etcd-operator-759567f574-zrhw7 1/1 Running 0 3d23h default cord-platform-etcd-operator-etcd-restore-operator-7bf97bd4hvzd9 1/1 Running 0 3d23h default cord-platform-grafana-85d4d648b9-chx2f 2/2 Running 0 3d23h default cord-platform-kafka-0 0/1 CrashLoopBackOff 929 3d23h default cord-platform-kibana-5f5d6f57c4-89dh5 1/1 Running 0 3d23h default cord-platform-logstash-0 1/1 Running 0 3d23h default cord-platform-onos-5b4bd8d876-6pfb7 2/2 Running 0 3d23h default cord-platform-prometheus-alertmanager-54c4d7b66c-lq6q6 2/2 Running 0 3d23h default cord-platform-prometheus-kube-state-metrics-85649df446-n8psv 1/1 Running 0 3d23h default cord-platform-prometheus-node-exporter-4fj2j 1/1 Running 0 3d23h default cord-platform-prometheus-node-exporter-mkjn4 1/1 Running 0 3d23h default cord-platform-prometheus-node-exporter-nhtpb 1/1 Running 0 3d23h default cord-platform-prometheus-pushgateway-68cdb5847-rwnjk 1/1 Running 0 3d23h default cord-platform-prometheus-server-9f9c454fb-bnhmc 2/2 Running 0 3d23h default cord-platform-zookeeper-0 1/1 Running 0 3d23h default etcd-cluster-6h2hfdkqvs 1/1 Running 0 4d default kpi-exporter-85c67bf99-4xrtf 1/1 Running 2 3d23h default kpi-exporter-85c67bf99-9nzpd 1/1 Running 2 3d23h default sadis-server-687855dbff-hxzhh 1/1 Running 0 3d23h default seba-base-kubernetes-tosca-loader-bxbj5 0/1 Completed 0 3d23h default seba-fabric-59b446c5df-2gwqs 1/1 Running 0 3d23h default seba-fabric-crossconnect-5c87866689-r44pk 1/1 Running 0 3d23h default seba-kubernetes-76d9dbf86b-7nwgw 1/1 Running 0 3d23h default seba-onos-service-649c575954-gv8j5 1/1 Running 1 3d23h default seba-rcord-5bb8d7bb68-bpg62 1/1 Running 0 3d23h default seba-seba-services-tosca-loader-lmhvj 0/1 Completed 0 3d23h default seba-volt-75d9466c4b-6sfwd 1/1 Running 0 3d23h default xos-chameleon-bc56d9cf9-zrkbx 1/1 Running 0 3d23h default xos-core-56ff8b89dd-bzjl4 1/1 Running 0 3d23h default xos-db-76f8db9cd8-t5vqx 1/1 Running 0 3d23h default xos-gui-6b658dd5df-ln7nr 1/1 Running 0 3d23h default xos-tosca-5b86cfb5bf-gqxck 1/1 Running 0 3d23h default xos-ws-d454bd6d4-mssns 1/1 Running 13 3d23h kube-system calico-kube-controllers-86b89b9949-nxxvt 1/1 Running 3 31d kube-system calico-node-9xsjt 1/1 Running 2 31d kube-system calico-node-d2fjb 1/1 Running 2 31d kube-system calico-node-zk44l 1/1 Running 2 31d kube-system coredns-97c4b444f-fvgzg 1/1 Running 1 31d kube-system coredns-97c4b444f-ngxtf 1/1 Running 2 31d kube-system dns-autoscaler-5fc5fdbf6-c48mk 1/1 Running 2 31d kube-system kube-apiserver-sbat1 1/1 Running 4 31d kube-system kube-apiserver-sbat2 1/1 Running 2 31d kube-system kube-apiserver-sbat3 1/1 Running 2 31d kube-system kube-controller-manager-sbat1 1/1 Running 5 31d kube-system kube-controller-manager-sbat2 1/1 Running 2 31d kube-system kube-controller-manager-sbat3 1/1 Running 2 31d kube-system kube-proxy-w8rlt 1/1 Running 2 31d kube-system kube-proxy-wr9gc 1/1 Running 2 31d kube-system kube-proxy-zcvq4 1/1 Running 3 31d kube-system kube-scheduler-sbat1 1/1 Running 5 31d kube-system kube-scheduler-sbat2 1/1 Running 4 31d kube-system kube-scheduler-sbat3 1/1 Running 3 31d kube-system kubernetes-dashboard-6c7466966c-2nl88 1/1 Running 2 31d kube-system nodelocaldns-c7ws4 1/1 Running 5 31d kube-system nodelocaldns-fgpmc 1/1 Running 2 31d kube-system nodelocaldns-kcp4h 1/1 Running 2 31d kube-system tiller-deploy-86c8b7897c-d72kq 1/1 Running 1 31d voltha default-http-backend-56ddf5b59-h8bgl 1/1 Running 0 3d23h voltha freeradius-5b469f597f-2852j 1/1 Running 0 3d23h voltha netconf-54974f6495-5mzsb 1/1 Running 0 3d23h voltha nginx-ingress-controller-59f977b6b4-tglws 1/1 Running 0 3d23h voltha ofagent-7b74788989-svbhv 1/1 Running 0 3d23h voltha vcli-65476cf646-bdqhs 1/1 Running 0 3d23h voltha vcore-0 1/1 Running 0 3d23h voltha voltha-5d76b54bcb-lbqfl 1/1 Running 0 3d23h
-
Story Points:1
-
Epic Link:
After returning to a running seba cluster after the weekend the kafka bus went into CrashLoopBackOff