Result | FAILURE |
Tests | 151 failed / 633 succeeded |
Started | |
Elapsed | 1h5m |
Revision | master |
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(block\svolmode\)\]\svolumes\sshould\sstore\sdata$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:159 Jan 17 17:37:39.314: failed: writing the contents: Unexpected error: <exec.CodeExitError>: { Err: { s: "error running /home/prow/go/src/k8s.io/kops/_rundir/5e76f245-9689-11ed-824d-f64c9135b4ea/kubectl --server=https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io --kubeconfig=/root/.kube/config --namespace=volume-6890 exec external-injector --namespace=volume-6890 -- /bin/sh -c echo 'Hello from ebs.csi.aws.com from namespace volume-6890' > /opt/0:\nCommand stdout:\n\nstderr:\nError from server: error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout\n\nerror:\nexit status 1", }, Code: 1, } error running /home/prow/go/src/k8s.io/kops/_rundir/5e76f245-9689-11ed-824d-f64c9135b4ea/kubectl --server=https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io --kubeconfig=/root/.kube/config --namespace=volume-6890 exec external-injector --namespace=volume-6890 -- /bin/sh -c echo 'Hello from ebs.csi.aws.com from namespace volume-6890' > /opt/0: Command stdout: stderr: Error from server: error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout error: exit status 1 occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/volume/fixtures.go:586from junit_01.xml
[BeforeEach] [Testpattern: Dynamic PV (block volmode)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (block volmode)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:36:58.636: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename volume �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should store data /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:159 Jan 17 17:36:58.849: INFO: Creating resource for dynamic PV Jan 17 17:36:58.849: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass volume-6890-e2e-sc6w9wj �[1mSTEP�[0m: creating a claim �[1mSTEP�[0m: starting external-injector �[1mSTEP�[0m: Writing text file contents in the container. Jan 17 17:37:09.041: INFO: Running '/home/prow/go/src/k8s.io/kops/_rundir/5e76f245-9689-11ed-824d-f64c9135b4ea/kubectl --server=https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io --kubeconfig=/root/.kube/config --namespace=volume-6890 exec external-injector --namespace=volume-6890 -- /bin/sh -c echo 'Hello from ebs.csi.aws.com from namespace volume-6890' > /opt/0' Jan 17 17:37:39.313: INFO: rc: 1 Jan 17 17:37:39.314: FAIL: failed: writing the contents: Unexpected error: <exec.CodeExitError>: { Err: { s: "error running /home/prow/go/src/k8s.io/kops/_rundir/5e76f245-9689-11ed-824d-f64c9135b4ea/kubectl --server=https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io --kubeconfig=/root/.kube/config --namespace=volume-6890 exec external-injector --namespace=volume-6890 -- /bin/sh -c echo 'Hello from ebs.csi.aws.com from namespace volume-6890' > /opt/0:\nCommand stdout:\n\nstderr:\nError from server: error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout\n\nerror:\nexit status 1", }, Code: 1, } error running /home/prow/go/src/k8s.io/kops/_rundir/5e76f245-9689-11ed-824d-f64c9135b4ea/kubectl --server=https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io --kubeconfig=/root/.kube/config --namespace=volume-6890 exec external-injector --namespace=volume-6890 -- /bin/sh -c echo 'Hello from ebs.csi.aws.com from namespace volume-6890' > /opt/0: Command stdout: stderr: Error from server: error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout error: exit status 1 occurred Full Stack Trace k8s.io/kubernetes/test/e2e/framework/volume.InjectContent(0xc000bcdce0, {{0xc0009dc730, 0xb}, {0x70d7126, 0x8}, {0x0, 0x0}, {0x0, 0x0, 0x0}, ...}, ...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/volume/fixtures.go:586 +0x7e7 k8s.io/kubernetes/test/e2e/storage/testsuites.(*volumesTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:186 +0x438 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000bac1a0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pod external-injector in namespace volume-6890 Jan 17 17:37:39.350: INFO: Waiting for pod external-injector to disappear Jan 17 17:37:39.380: INFO: Pod external-injector still exists Jan 17 17:37:41.382: INFO: Waiting for pod external-injector to disappear Jan 17 17:37:41.412: INFO: Pod external-injector still exists Jan 17 17:37:43.381: INFO: Waiting for pod external-injector to disappear Jan 17 17:37:43.411: INFO: Pod external-injector no longer exists �[1mSTEP�[0m: cleaning the environment after external �[1mSTEP�[0m: Deleting pvc Jan 17 17:37:43.472: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comdn2sg" Jan 17 17:37:43.504: INFO: Waiting up to 3m0s for PersistentVolume pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103 to get deleted Jan 17 17:37:43.535: INFO: PersistentVolume pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103 found and phase=Released (30.815567ms) Jan 17 17:37:48.566: INFO: PersistentVolume pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103 found and phase=Released (5.061832411s) Jan 17 17:37:53.600: INFO: PersistentVolume pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103 was removed �[1mSTEP�[0m: Deleting sc [AfterEach] [Testpattern: Dynamic PV (block volmode)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "volume-6890". �[1mSTEP�[0m: Found 12 events. Jan 17 17:37:53.663: INFO: At 2023-01-17 17:36:58 +0000 UTC - event for ebs.csi.aws.comdn2sg: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:37:53.663: INFO: At 2023-01-17 17:36:58 +0000 UTC - event for ebs.csi.aws.comdn2sg: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "volume-6890/ebs.csi.aws.comdn2sg" Jan 17 17:37:53.663: INFO: At 2023-01-17 17:36:58 +0000 UTC - event for ebs.csi.aws.comdn2sg: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:02 +0000 UTC - event for ebs.csi.aws.comdn2sg: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103 Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:02 +0000 UTC - event for external-injector: {default-scheduler } Scheduled: Successfully assigned volume-6890/external-injector to ip-172-20-41-188.ec2.internal Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:05 +0000 UTC - event for external-injector: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103" Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:05 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} SuccessfulMountVolume: MapVolume.MapPodDevice succeeded for volume "pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103" globalMapPath "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103/dev" Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:05 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} SuccessfulMountVolume: MapVolume.MapPodDevice succeeded for volume "pvc-f4b2dff5-e5c7-4c63-92cb-29c2299ea103" volumeMapPath "/var/lib/kubelet/pods/73c40e87-f8f3-4ace-bbc0-f3fc66e3b6df/volumeDevices/kubernetes.io~csi" Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:06 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" already present on machine Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:06 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} Created: Created container external-injector Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:06 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} Started: Started container external-injector Jan 17 17:37:53.663: INFO: At 2023-01-17 17:37:39 +0000 UTC - event for external-injector: {kubelet ip-172-20-41-188.ec2.internal} Killing: Stopping container external-injector Jan 17 17:37:53.693: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:37:53.694: INFO: Jan 17 17:37:53.725: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:37:53.755: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 12979 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:36:00 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:11 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:11 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:11 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:37:11 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:37:53.756: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:37:53.788: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:38:23.819: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:38:23.819: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:38:23.850: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 12764 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:36:50 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:38:23.850: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:38:23.883: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:38:53.915: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:38:53.915: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:38:53.945: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 14600 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-8696":"ip-172-20-41-188.ec2.internal","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:38:14 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status} {kube-controller-manager Update v1 2023-01-17 17:38:49 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:38:14 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:38:14 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:38:14 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:38:14 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-8696^cc89cf39-968d-11ed-993d-42ca828f3c18,DevicePath:,},},Config:nil,},} Jan 17 17:38:53.946: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:38:53.982: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:39:24.013: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:39:24.013: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:39:24.044: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 14948 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-48-3.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-8482":"ip-172-20-48-3.ec2.internal","ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kubelet Update v1 2023-01-17 17:39:02 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:39:02 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:39:02 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:39:02 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:39:02 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:39:24.045: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:39:24.078: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:39:54.109: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:39:54.109: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:39:54.139: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 12964 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:39:54.140: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:39:54.172: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:39:54.208: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.208: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:39:54.208: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:39:54.208: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:39:54.209: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:39:54.209: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:39:54.209: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:39:54.209: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:39:54.209: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:39:54.209: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:39:54.209: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:39:54.209: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:39:54.209: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:39:54.209: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:39:54.209: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:39:54.209: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:39:54.209: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:39:54.319: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:39:54.320: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "volume-6890" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\sfsgroupchangepolicy\s\(Always\)\[LinuxOnly\]\,\spod\screated\swith\san\sinitial\sfsgroup\,\snew\spod\sfsgroup\sapplied\sto\svolume\scontents$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:28:54.640: Unexpected error: <*errors.StatusError | 0xc000dfc3c0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270from junit_19.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:28:12.114: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename fsgroupchangepolicy �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] (Always)[LinuxOnly], pod created with an initial fsgroup, new pod fsgroup applied to volume contents /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:28:12.310: INFO: Creating resource for dynamic PV Jan 17 17:28:12.310: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass fsgroupchangepolicy-5353-e2e-scvqrgj �[1mSTEP�[0m: creating a claim Jan 17 17:28:12.341: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating Pod in namespace fsgroupchangepolicy-5353 with fsgroup 1000 Jan 17 17:28:24.521: INFO: Pod fsgroupchangepolicy-5353/pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e started successfully �[1mSTEP�[0m: Creating a sub-directory and file, and verifying their ownership is 1000 Jan 17 17:28:24.522: INFO: ExecWithOptions {Command:[/bin/sh -c touch /mnt/volume1/file1] Namespace:fsgroupchangepolicy-5353 PodName:pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e ContainerName:write-pod Stdin:<nil> CaptureStdout:true CaptureStderr:true PreserveWhitespace:false Quiet:false} Jan 17 17:28:24.522: INFO: >>> kubeConfig: /root/.kube/config Jan 17 17:28:24.523: INFO: ExecWithOptions: Clientset creation Jan 17 17:28:24.523: INFO: ExecWithOptions: execute(POST https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io/api/v1/namespaces/fsgroupchangepolicy-5353/pods/pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e/exec?command=%2Fbin%2Fsh&command=-c&command=touch+%2Fmnt%2Fvolume1%2Ffile1&container=write-pod&container=write-pod&stderr=true&stdout=true %!s(MISSING)) Jan 17 17:28:54.640: FAIL: Unexpected error: <*errors.StatusError | 0xc000dfc3c0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc00155f4a0, 0xc0047cf070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc00088eea0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pvc Jan 17 17:28:54.697: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.com5sqmz" Jan 17 17:28:54.726: INFO: Waiting up to 3m0s for PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 to get deleted Jan 17 17:28:54.753: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (27.689627ms) Jan 17 17:28:59.783: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (5.057270203s) Jan 17 17:29:04.812: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (10.086010264s) Jan 17 17:29:09.841: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (15.115318228s) Jan 17 17:29:14.877: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (20.15104962s) Jan 17 17:29:19.906: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (25.180220026s) Jan 17 17:29:24.935: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (30.20887021s) Jan 17 17:29:29.965: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (35.238862542s) Jan 17 17:29:34.994: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (40.268160997s) Jan 17 17:29:40.023: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (45.297162955s) Jan 17 17:29:45.052: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (50.326446922s) Jan 17 17:29:50.081: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (55.35518291s) Jan 17 17:29:55.110: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m0.384089513s) Jan 17 17:30:00.139: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m5.413575199s) Jan 17 17:30:05.168: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m10.442518586s) Jan 17 17:30:10.197: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m15.471226093s) Jan 17 17:30:15.227: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m20.501466904s) Jan 17 17:30:20.256: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m25.530296574s) Jan 17 17:30:25.287: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m30.561061553s) Jan 17 17:30:30.316: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m35.589865393s) Jan 17 17:30:35.345: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m40.619372769s) Jan 17 17:30:40.376: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m45.649928169s) Jan 17 17:30:45.407: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m50.681147948s) Jan 17 17:30:50.436: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (1m55.709865626s) Jan 17 17:30:55.465: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m0.738969218s) Jan 17 17:31:00.494: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m5.767827078s) Jan 17 17:31:05.522: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m10.796464941s) Jan 17 17:31:10.551: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m15.825719615s) Jan 17 17:31:15.580: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m20.853843914s) Jan 17 17:31:20.609: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m25.882930605s) Jan 17 17:31:25.638: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m30.912166886s) Jan 17 17:31:30.667: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m35.940979855s) Jan 17 17:31:35.695: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m40.969665416s) Jan 17 17:31:40.724: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m45.998418077s) Jan 17 17:31:45.753: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m51.027078809s) Jan 17 17:31:50.781: INFO: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 found and phase=Bound (2m56.055610861s) �[1mSTEP�[0m: Deleting sc Jan 17 17:31:55.812: FAIL: while cleanup resource Unexpected error: <errors.aggregate | len:1, cap:1>: [ [ { msg: "persistent Volume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 not deleted by dynamic provisioner: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 still exists within 3m0s", err: { s: "PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 still exists within 3m0s", }, }, ], ] persistent Volume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 not deleted by dynamic provisioner: PersistentVolume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 still exists within 3m0s occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func2() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:132 +0x20f panic({0x6c3d000, 0xc0031c0900}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail.func1() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:63 +0x73 panic({0x62d5960, 0x78abec0}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail({0xc00161a000, 0x26a}, {0xc0047ced30, 0x70cbf6a, 0xc0047ced50}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:67 +0x1a7 k8s.io/kubernetes/test/e2e/framework.Fail({0xc0041f7b80, 0x255}, {0xc002332260, 0xc0041f7b80, 0xc0047cee78}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/log.go:63 +0x149 k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).match(0xc0047cee98, {0x79beb98, 0xaa12408}, 0x0, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:79 +0x1bd k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).NotTo(0xc0047cee98, {0x79beb98, 0xaa12408}, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:48 +0x92 k8s.io/kubernetes/test/e2e/framework.ExpectNoErrorWithOffset(0x70d7f8e, {0x78ba840, 0xc000dfc3c0}, {0x0, 0x1, 0x2}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:46 +0xa9 k8s.io/kubernetes/test/e2e/framework.ExpectNoError(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:40 k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc00155f4a0, 0xc0047cf070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc00088eea0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "fsgroupchangepolicy-5353". �[1mSTEP�[0m: Found 9 events. Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:12 +0000 UTC - event for ebs.csi.aws.com5sqmz: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:12 +0000 UTC - event for ebs.csi.aws.com5sqmz: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:12 +0000 UTC - event for ebs.csi.aws.com5sqmz: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "fsgroupchangepolicy-5353/ebs.csi.aws.com5sqmz" Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:15 +0000 UTC - event for ebs.csi.aws.com5sqmz: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8 Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:16 +0000 UTC - event for pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e: {default-scheduler } Scheduled: Successfully assigned fsgroupchangepolicy-5353/pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e to ip-172-20-34-144.ec2.internal Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:18 +0000 UTC - event for pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-a6e039c9-fcff-4b6c-9a49-4b5c245106b8" Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:22 +0000 UTC - event for pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e: {kubelet ip-172-20-34-144.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" already present on machine Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:22 +0000 UTC - event for pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e: {kubelet ip-172-20-34-144.ec2.internal} Created: Created container write-pod Jan 17 17:31:55.842: INFO: At 2023-01-17 17:28:23 +0000 UTC - event for pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e: {kubelet ip-172-20-34-144.ec2.internal} Started: Started container write-pod Jan 17 17:31:55.870: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:31:55.870: INFO: pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e ip-172-20-34-144.ec2.internal Running [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:16 +0000 UTC } {Ready True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:23 +0000 UTC } {ContainersReady True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:23 +0000 UTC } {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:16 +0000 UTC }] Jan 17 17:31:55.870: INFO: Jan 17 17:32:25.900: INFO: Unable to fetch fsgroupchangepolicy-5353/pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e/write-pod logs: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-e0b2d524-73ba-4a60-bf31-77f2dda7279e) Jan 17 17:32:25.930: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:32:25.958: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 9408 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-1546":"ip-172-20-32-111.ec2.internal","csi-mock-csi-mock-volumes-3354":"csi-mock-csi-mock-volumes-3354","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:32:00 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:32:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-1546^d86a7a90-968c-11ed-8bbb-269df89f2081],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-1546^d86a7a90-968c-11ed-8bbb-269df89f2081,DevicePath:,},},Config:nil,},} Jan 17 17:32:25.959: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:32:26.009: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:32:56.057: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:32:56.057: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:32:56.086: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 9924 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:51 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:25:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e,DevicePath:,},},Config:nil,},} Jan 17 17:32:56.086: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:32:56.116: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:33:26.145: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:33:26.145: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:33:26.173: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 9938 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:32:47 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:33:26.173: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:33:26.205: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:33:56.234: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:33:56.234: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:33:56.263: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 9812 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kubelet Update v1 2023-01-17 17:29:00 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:33:56.263: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:33:56.295: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:34:26.323: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:34:26.323: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:34:26.351: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 9382 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:34:26.352: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:34:26.382: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:34:26.447: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:34:26.447: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:34:26.447: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:34:26.447: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:34:26.447: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:34:26.447: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:34:26.447: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:34:26.447: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:34:26.447: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:34:26.447: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:34:26.447: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:34:26.447: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:34:26.447: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:34:26.447: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:34:26.447: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:26.447: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:34:26.569: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:34:26.569: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "fsgroupchangepolicy-5353" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\sfsgroupchangepolicy\s\(Always\)\[LinuxOnly\]\,\spod\screated\swith\san\sinitial\sfsgroup\,\svolume\scontents\sownership\schanged\svia\schgrp\sin\sfirst\spod\,\snew\spod\swith\sdifferent\sfsgroup\sapplied\sto\sthe\svolume\scontents$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:29:03.315: Unexpected error: <*errors.StatusError | 0xc000a185a0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270from junit_24.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:28:18.767: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename fsgroupchangepolicy �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] (Always)[LinuxOnly], pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with different fsgroup applied to the volume contents /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:28:18.981: INFO: Creating resource for dynamic PV Jan 17 17:28:18.981: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass fsgroupchangepolicy-706-e2e-scbjvw9 �[1mSTEP�[0m: creating a claim Jan 17 17:28:19.014: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating Pod in namespace fsgroupchangepolicy-706 with fsgroup 1000 Jan 17 17:28:33.203: INFO: Pod fsgroupchangepolicy-706/pod-b676b5be-de67-4677-ad8b-14b9579db5ba started successfully �[1mSTEP�[0m: Creating a sub-directory and file, and verifying their ownership is 1000 Jan 17 17:28:33.203: INFO: ExecWithOptions {Command:[/bin/sh -c touch /mnt/volume1/file1] Namespace:fsgroupchangepolicy-706 PodName:pod-b676b5be-de67-4677-ad8b-14b9579db5ba ContainerName:write-pod Stdin:<nil> CaptureStdout:true CaptureStderr:true PreserveWhitespace:false Quiet:false} Jan 17 17:28:33.203: INFO: >>> kubeConfig: /root/.kube/config Jan 17 17:28:33.204: INFO: ExecWithOptions: Clientset creation Jan 17 17:28:33.204: INFO: ExecWithOptions: execute(POST https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io/api/v1/namespaces/fsgroupchangepolicy-706/pods/pod-b676b5be-de67-4677-ad8b-14b9579db5ba/exec?command=%2Fbin%2Fsh&command=-c&command=touch+%2Fmnt%2Fvolume1%2Ffile1&container=write-pod&container=write-pod&stderr=true&stdout=true %!s(MISSING)) Jan 17 17:29:03.315: FAIL: Unexpected error: <*errors.StatusError | 0xc000a185a0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.34.144:10250: i/o timeout occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc0034c54a0, 0xc002719070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000b24000, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pvc Jan 17 17:29:03.376: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comlfg77" Jan 17 17:29:03.409: INFO: Waiting up to 3m0s for PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc to get deleted Jan 17 17:29:03.439: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (30.11342ms) Jan 17 17:29:08.470: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (5.060916945s) Jan 17 17:29:13.504: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (10.094821057s) Jan 17 17:29:18.535: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (15.12536683s) Jan 17 17:29:23.567: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (20.15787333s) Jan 17 17:29:28.598: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (25.189136584s) Jan 17 17:29:33.631: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (30.221752708s) Jan 17 17:29:38.665: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (35.255738469s) Jan 17 17:29:43.699: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (40.28977334s) Jan 17 17:29:48.730: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (45.320791601s) Jan 17 17:29:53.761: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (50.352140552s) Jan 17 17:29:58.796: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (55.386309139s) Jan 17 17:30:03.827: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m0.417911173s) Jan 17 17:30:08.861: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m5.451813612s) Jan 17 17:30:13.895: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m10.485949346s) Jan 17 17:30:18.926: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m15.516832099s) Jan 17 17:30:23.960: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m20.551161524s) Jan 17 17:30:28.991: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m25.582079971s) Jan 17 17:30:34.022: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m30.612740913s) Jan 17 17:30:39.055: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m35.645331776s) Jan 17 17:30:44.089: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m40.680026887s) Jan 17 17:30:49.124: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m45.714397941s) Jan 17 17:30:54.158: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m50.748586415s) Jan 17 17:30:59.189: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (1m55.779456469s) Jan 17 17:31:04.220: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m0.81041282s) Jan 17 17:31:09.251: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m5.84173293s) Jan 17 17:31:14.282: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m10.872330194s) Jan 17 17:31:19.313: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m15.903646148s) Jan 17 17:31:24.344: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m20.934351903s) Jan 17 17:31:29.375: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m25.965598443s) Jan 17 17:31:34.405: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m30.995974669s) Jan 17 17:31:39.436: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m36.02687832s) Jan 17 17:31:44.467: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m41.057705385s) Jan 17 17:31:49.498: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m46.088702835s) Jan 17 17:31:54.531: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m51.121340382s) Jan 17 17:31:59.572: INFO: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc found and phase=Bound (2m56.163252434s) �[1mSTEP�[0m: Deleting sc Jan 17 17:32:04.607: FAIL: while cleanup resource Unexpected error: <errors.aggregate | len:1, cap:1>: [ [ { msg: "persistent Volume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc not deleted by dynamic provisioner: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc still exists within 3m0s", err: { s: "PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc still exists within 3m0s", }, }, ], ] persistent Volume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc not deleted by dynamic provisioner: PersistentVolume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc still exists within 3m0s occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func2() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:132 +0x20f panic({0x6c3d000, 0xc004130880}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail.func1() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:63 +0x73 panic({0x62d5960, 0x78abec0}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail({0xc001ef6280, 0x26a}, {0xc0026a4d30, 0x70cbf6a, 0xc0026a4d50}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:67 +0x1a7 k8s.io/kubernetes/test/e2e/framework.Fail({0xc001ef6000, 0x255}, {0xc001c50d80, 0xc001ef6000, 0xc0026a4e78}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/log.go:63 +0x149 k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).match(0xc0026a4e98, {0x79beb98, 0xaa12408}, 0x0, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:79 +0x1bd k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).NotTo(0xc0026a4e98, {0x79beb98, 0xaa12408}, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:48 +0x92 k8s.io/kubernetes/test/e2e/framework.ExpectNoErrorWithOffset(0x70d7f8e, {0x78ba840, 0xc000a185a0}, {0x0, 0x1, 0x2}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:46 +0xa9 k8s.io/kubernetes/test/e2e/framework.ExpectNoError(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:40 k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc0034c54a0, 0xc002719070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000b24000, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "fsgroupchangepolicy-706". �[1mSTEP�[0m: Found 9 events. Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:19 +0000 UTC - event for ebs.csi.aws.comlfg77: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:19 +0000 UTC - event for ebs.csi.aws.comlfg77: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:19 +0000 UTC - event for ebs.csi.aws.comlfg77: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "fsgroupchangepolicy-706/ebs.csi.aws.comlfg77" Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:22 +0000 UTC - event for ebs.csi.aws.comlfg77: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-32b6ab0e-d221-454e-94e9-246ac881bedc Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:23 +0000 UTC - event for pod-b676b5be-de67-4677-ad8b-14b9579db5ba: {default-scheduler } Scheduled: Successfully assigned fsgroupchangepolicy-706/pod-b676b5be-de67-4677-ad8b-14b9579db5ba to ip-172-20-34-144.ec2.internal Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:25 +0000 UTC - event for pod-b676b5be-de67-4677-ad8b-14b9579db5ba: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-32b6ab0e-d221-454e-94e9-246ac881bedc" Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:32 +0000 UTC - event for pod-b676b5be-de67-4677-ad8b-14b9579db5ba: {kubelet ip-172-20-34-144.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" already present on machine Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:32 +0000 UTC - event for pod-b676b5be-de67-4677-ad8b-14b9579db5ba: {kubelet ip-172-20-34-144.ec2.internal} Created: Created container write-pod Jan 17 17:32:04.638: INFO: At 2023-01-17 17:28:32 +0000 UTC - event for pod-b676b5be-de67-4677-ad8b-14b9579db5ba: {kubelet ip-172-20-34-144.ec2.internal} Started: Started container write-pod Jan 17 17:32:04.669: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:32:04.669: INFO: pod-b676b5be-de67-4677-ad8b-14b9579db5ba ip-172-20-34-144.ec2.internal Running [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:23 +0000 UTC } {Ready True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:32 +0000 UTC } {ContainersReady True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:32 +0000 UTC } {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:28:23 +0000 UTC }] Jan 17 17:32:04.669: INFO: Jan 17 17:32:34.699: INFO: Unable to fetch fsgroupchangepolicy-706/pod-b676b5be-de67-4677-ad8b-14b9579db5ba/write-pod logs: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-b676b5be-de67-4677-ad8b-14b9579db5ba) Jan 17 17:32:34.730: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:32:34.761: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 9408 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-1546":"ip-172-20-32-111.ec2.internal","csi-mock-csi-mock-volumes-3354":"csi-mock-csi-mock-volumes-3354","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:32:00 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:32:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:06 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-1546^d86a7a90-968c-11ed-8bbb-269df89f2081],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-1546^d86a7a90-968c-11ed-8bbb-269df89f2081,DevicePath:,},},Config:nil,},} Jan 17 17:32:34.761: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:32:34.793: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:33:04.824: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:33:04.824: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:33:04.854: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 9924 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:51 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:25:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e,DevicePath:,},},Config:nil,},} Jan 17 17:33:04.855: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:33:04.888: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:33:34.918: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:33:34.919: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:33:34.949: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 9938 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:32:47 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:47 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:33:34.950: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:33:34.983: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:34:05.013: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:34:05.013: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:34:05.043: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 9812 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kubelet Update v1 2023-01-17 17:29:00 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:35 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:34:05.044: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:34:05.079: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:34:35.110: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:34:35.110: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:34:35.141: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 9382 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:34:35.141: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:34:35.174: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:34:35.209: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:34:35.209: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:34:35.209: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:34:35.209: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:34:35.209: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:34:35.209: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:34:35.209: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:34:35.209: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:34:35.209: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:34:35.209: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:34:35.209: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:34:35.209: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:34:35.209: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:34:35.209: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:34:35.209: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:34:35.209: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:34:35.318: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:34:35.318: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "fsgroupchangepolicy-706" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\sfsgroupchangepolicy\s\(OnRootMismatch\)\[LinuxOnly\]\,\spod\screated\swith\san\sinitial\sfsgroup\,\snew\spod\sfsgroup\sapplied\sto\svolume\scontents$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:23:18.079: Unexpected error: <*errors.StatusError | 0xc0045bc5a0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.48.3:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.48.3:10250: i/o timeout occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270from junit_11.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:22:29.042: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename fsgroupchangepolicy W0117 17:22:29.239958 6806 warnings.go:70] policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Jan 17 17:22:29.240: INFO: No PodSecurityPolicies found; assuming PodSecurityPolicy is disabled. �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] (OnRootMismatch)[LinuxOnly], pod created with an initial fsgroup, new pod fsgroup applied to volume contents /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:22:29.366: INFO: Creating resource for dynamic PV Jan 17 17:22:29.366: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass fsgroupchangepolicy-1767-e2e-scmqfsx �[1mSTEP�[0m: creating a claim Jan 17 17:22:29.483: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating Pod in namespace fsgroupchangepolicy-1767 with fsgroup 1000 Jan 17 17:22:47.968: INFO: Pod fsgroupchangepolicy-1767/pod-8b46d04c-1361-4641-b020-a788d1ba229f started successfully �[1mSTEP�[0m: Creating a sub-directory and file, and verifying their ownership is 1000 Jan 17 17:22:47.968: INFO: ExecWithOptions {Command:[/bin/sh -c touch /mnt/volume1/file1] Namespace:fsgroupchangepolicy-1767 PodName:pod-8b46d04c-1361-4641-b020-a788d1ba229f ContainerName:write-pod Stdin:<nil> CaptureStdout:true CaptureStderr:true PreserveWhitespace:false Quiet:false} Jan 17 17:22:47.968: INFO: >>> kubeConfig: /root/.kube/config Jan 17 17:22:47.969: INFO: ExecWithOptions: Clientset creation Jan 17 17:22:47.969: INFO: ExecWithOptions: execute(POST https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io/api/v1/namespaces/fsgroupchangepolicy-1767/pods/pod-8b46d04c-1361-4641-b020-a788d1ba229f/exec?command=%2Fbin%2Fsh&command=-c&command=touch+%2Fmnt%2Fvolume1%2Ffile1&container=write-pod&container=write-pod&stderr=true&stdout=true %!s(MISSING)) Jan 17 17:23:18.079: FAIL: Unexpected error: <*errors.StatusError | 0xc0045bc5a0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.48.3:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.48.3:10250: i/o timeout occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc0028e0000, 0xc0041b3070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000981040, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pvc Jan 17 17:23:18.140: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.com7gxdb" Jan 17 17:23:18.174: INFO: Waiting up to 3m0s for PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab to get deleted Jan 17 17:23:18.205: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (30.687735ms) Jan 17 17:23:23.235: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (5.061224909s) Jan 17 17:23:28.266: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (10.0921857s) Jan 17 17:23:33.299: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (15.125393067s) Jan 17 17:23:38.331: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (20.157422271s) Jan 17 17:23:43.365: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (25.190697548s) Jan 17 17:23:48.398: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (30.224173497s) Jan 17 17:23:53.429: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (35.255115833s) Jan 17 17:23:58.460: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (40.286066436s) Jan 17 17:24:03.492: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (45.318342367s) Jan 17 17:24:08.523: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (50.349157683s) Jan 17 17:24:13.554: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (55.379860295s) Jan 17 17:24:18.588: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m0.413956414s) Jan 17 17:24:23.625: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m5.450582709s) Jan 17 17:24:28.656: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m10.482166452s) Jan 17 17:24:33.687: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m15.512592663s) Jan 17 17:24:38.718: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m20.543499581s) Jan 17 17:24:43.748: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m25.573963187s) Jan 17 17:24:48.782: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m30.608240773s) Jan 17 17:24:53.818: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m35.644008933s) Jan 17 17:24:58.852: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m40.678417592s) Jan 17 17:25:03.887: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m45.712509225s) Jan 17 17:25:08.920: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m50.745961824s) Jan 17 17:25:13.951: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (1m55.777250744s) Jan 17 17:25:18.986: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m0.811484851s) Jan 17 17:25:24.017: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m5.842821058s) Jan 17 17:25:29.048: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m10.873941532s) Jan 17 17:25:34.079: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m15.904681232s) Jan 17 17:25:39.113: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m20.938809724s) Jan 17 17:25:44.147: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m25.973371331s) Jan 17 17:25:49.180: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m31.006035576s) Jan 17 17:25:54.211: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m36.036983685s) Jan 17 17:25:59.242: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m41.067862925s) Jan 17 17:26:04.275: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m46.101288774s) Jan 17 17:26:09.308: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m51.134100751s) Jan 17 17:26:14.339: INFO: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab found and phase=Bound (2m56.164784952s) �[1mSTEP�[0m: Deleting sc Jan 17 17:26:19.371: FAIL: while cleanup resource Unexpected error: <errors.aggregate | len:1, cap:1>: [ [ { msg: "persistent Volume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab not deleted by dynamic provisioner: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab still exists within 3m0s", err: { s: "PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab still exists within 3m0s", }, }, ], ] persistent Volume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab not deleted by dynamic provisioner: PersistentVolume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab still exists within 3m0s occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func2() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:132 +0x20f panic({0x6c3d000, 0xc004030b80}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail.func1() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:63 +0x73 panic({0x62d5960, 0x78abec0}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail({0xc000633680, 0x266}, {0xc0041b2d30, 0x70cbf6a, 0xc0041b2d50}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:67 +0x1a7 k8s.io/kubernetes/test/e2e/framework.Fail({0xc000633400, 0x251}, {0xc004685460, 0xc000633400, 0xc0041b2e78}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/log.go:63 +0x149 k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).match(0xc0041b2e98, {0x79beb98, 0xaa12408}, 0x0, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:79 +0x1bd k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).NotTo(0xc0041b2e98, {0x79beb98, 0xaa12408}, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:48 +0x92 k8s.io/kubernetes/test/e2e/framework.ExpectNoErrorWithOffset(0x70d7f8e, {0x78ba840, 0xc0045bc5a0}, {0x0, 0x1, 0x2}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:46 +0xa9 k8s.io/kubernetes/test/e2e/framework.ExpectNoError(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:40 k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc0028e0000, 0xc0041b3070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000981040, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "fsgroupchangepolicy-1767". �[1mSTEP�[0m: Found 10 events. Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:29 +0000 UTC - event for ebs.csi.aws.com7gxdb: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:29 +0000 UTC - event for ebs.csi.aws.com7gxdb: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:29 +0000 UTC - event for ebs.csi.aws.com7gxdb: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "fsgroupchangepolicy-1767/ebs.csi.aws.com7gxdb" Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:33 +0000 UTC - event for ebs.csi.aws.com7gxdb: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-dee96530-b2fb-46c5-9973-7307cf8201ab Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:33 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {default-scheduler } Scheduled: Successfully assigned fsgroupchangepolicy-1767/pod-8b46d04c-1361-4641-b020-a788d1ba229f to ip-172-20-48-3.ec2.internal Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:38 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-dee96530-b2fb-46c5-9973-7307cf8201ab" Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:45 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {kubelet ip-172-20-48-3.ec2.internal} Pulling: Pulling image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:45 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {kubelet ip-172-20-48-3.ec2.internal} Pulled: Successfully pulled image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" in 474.112437ms Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:45 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {kubelet ip-172-20-48-3.ec2.internal} Created: Created container write-pod Jan 17 17:26:19.403: INFO: At 2023-01-17 17:22:45 +0000 UTC - event for pod-8b46d04c-1361-4641-b020-a788d1ba229f: {kubelet ip-172-20-48-3.ec2.internal} Started: Started container write-pod Jan 17 17:26:19.434: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:26:19.434: INFO: pod-8b46d04c-1361-4641-b020-a788d1ba229f ip-172-20-48-3.ec2.internal Running [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:22:33 +0000 UTC } {Ready True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:22:45 +0000 UTC } {ContainersReady True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:22:45 +0000 UTC } {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:22:33 +0000 UTC }] Jan 17 17:26:19.434: INFO: Jan 17 17:26:49.465: INFO: Unable to fetch fsgroupchangepolicy-1767/pod-8b46d04c-1361-4641-b020-a788d1ba229f/write-pod logs: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-8b46d04c-1361-4641-b020-a788d1ba229f) Jan 17 17:26:49.526: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:26:49.556: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 3973 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:41 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:22:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-03fdd1fbca2d05e64],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-03fdd1fbca2d05e64,DevicePath:,},},Config:nil,},} Jan 17 17:26:49.557: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:26:49.589: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:27:19.620: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:27:19.620: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:27:19.651: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 5905 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:51 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:25:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:18 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:18 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:18 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:26:18 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0cc170227dc73d69e],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0cc170227dc73d69e,DevicePath:,},},Config:nil,},} Jan 17 17:27:19.651: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:27:19.683: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:27:49.724: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:27:49.724: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:27:49.755: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 6140 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-6801":"ip-172-20-41-188.ec2.internal","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:35 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:27:11 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:27:40 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:27:40 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:27:40 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:27:40 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-04ab1ed8f9d623bd6 kubernetes.io/csi/ebs.csi.aws.com^vol-05a00ff762da58a3e kubernetes.io/csi/ebs.csi.aws.com^vol-0c6f86666abaacbe2],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-04ab1ed8f9d623bd6,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0c6f86666abaacbe2,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-05a00ff762da58a3e,DevicePath:,},},Config:nil,},} Jan 17 17:27:49.755: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:27:49.788: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:28:19.819: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:28:19.819: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:28:19.849: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 5534 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:22:37 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:22:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:47 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:47 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:47 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:26:47 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0ab9adf04b2689417],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0ab9adf04b2689417,DevicePath:,},},Config:nil,},} Jan 17 17:28:19.849: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:28:19.882: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:28:49.913: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:28:49.913: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:28:49.943: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 5648 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:57 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:57 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:26:57 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:26:57 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:28:49.944: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:28:49.979: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:28:50.045: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:28:50.045: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:28:50.045: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:28:50.045: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:28:50.045: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:28:50.045: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:28:50.045: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:28:50.045: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:28:50.045: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:28:50.045: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:28:50.045: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:28:50.045: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:28:50.045: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:28:50.045: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:28:50.045: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:28:50.045: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:28:50.167: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:28:50.167: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "fsgroupchangepolicy-1767" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\sfsgroupchangepolicy\s\(OnRootMismatch\)\[LinuxOnly\]\,\spod\screated\swith\san\sinitial\sfsgroup\,\svolume\scontents\sownership\schanged\svia\schgrp\sin\sfirst\spod\,\snew\spod\swith\ssame\sfsgroup\sskips\sownership\schanges\sto\sthe\svolume\scontents$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:27:07.482: Unexpected error: <*errors.StatusError | 0xc003f44640>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270from junit_16.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:26:24.973: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename fsgroupchangepolicy �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] (OnRootMismatch)[LinuxOnly], pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with same fsgroup skips ownership changes to the volume contents /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:214 Jan 17 17:26:25.173: INFO: Creating resource for dynamic PV Jan 17 17:26:25.173: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass fsgroupchangepolicy-8172-e2e-sclwb2r �[1mSTEP�[0m: creating a claim Jan 17 17:26:25.203: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating Pod in namespace fsgroupchangepolicy-8172 with fsgroup 1000 Jan 17 17:26:37.380: INFO: Pod fsgroupchangepolicy-8172/pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc started successfully �[1mSTEP�[0m: Creating a sub-directory and file, and verifying their ownership is 1000 Jan 17 17:26:37.380: INFO: ExecWithOptions {Command:[/bin/sh -c touch /mnt/volume1/file1] Namespace:fsgroupchangepolicy-8172 PodName:pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc ContainerName:write-pod Stdin:<nil> CaptureStdout:true CaptureStderr:true PreserveWhitespace:false Quiet:false} Jan 17 17:26:37.380: INFO: >>> kubeConfig: /root/.kube/config Jan 17 17:26:37.381: INFO: ExecWithOptions: Clientset creation Jan 17 17:26:37.381: INFO: ExecWithOptions: execute(POST https://api.e2e-e2e-kops-grid-cilium-eni-rhel8-k23-ko26-docker.test-cncf-aws.k8s.io/api/v1/namespaces/fsgroupchangepolicy-8172/pods/pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc/exec?command=%2Fbin%2Fsh&command=-c&command=touch+%2Fmnt%2Fvolume1%2Ffile1&container=write-pod&container=write-pod&stderr=true&stdout=true %!s(MISSING)) Jan 17 17:27:07.482: FAIL: Unexpected error: <*errors.StatusError | 0xc003f44640>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout", Reason: "", Details: nil, Code: 500, }, } error dialing backend: dial tcp 172.20.41.188:10250: i/o timeout occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc001b90c60, 0xc003cdb070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc0009c81a0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pvc Jan 17 17:27:07.540: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comx4lzr" Jan 17 17:27:07.570: INFO: Waiting up to 3m0s for PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 to get deleted Jan 17 17:27:07.598: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (28.042936ms) Jan 17 17:27:12.630: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (5.059880258s) Jan 17 17:27:17.659: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (10.088857662s) Jan 17 17:27:22.688: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (15.117757195s) Jan 17 17:27:27.717: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (20.14727893s) Jan 17 17:27:32.746: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (25.176426798s) Jan 17 17:27:37.777: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (30.20704117s) Jan 17 17:27:42.806: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (35.235750726s) Jan 17 17:27:47.835: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (40.265071855s) Jan 17 17:27:52.864: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (45.294073171s) Jan 17 17:27:57.895: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (50.325608482s) Jan 17 17:28:02.927: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (55.356774206s) Jan 17 17:28:07.958: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m0.388298914s) Jan 17 17:28:12.989: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m5.419365144s) Jan 17 17:28:18.019: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m10.44907786s) Jan 17 17:28:23.049: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m15.479614149s) Jan 17 17:28:28.082: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m20.511727343s) Jan 17 17:28:33.118: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m25.548186781s) Jan 17 17:28:38.147: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m30.57739554s) Jan 17 17:28:43.178: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m35.608190158s) Jan 17 17:28:48.209: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m40.63934229s) Jan 17 17:28:53.241: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m45.67085596s) Jan 17 17:28:58.270: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m50.700006419s) Jan 17 17:29:03.299: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (1m55.729406241s) Jan 17 17:29:08.330: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m0.759921318s) Jan 17 17:29:13.358: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m5.788604669s) Jan 17 17:29:18.392: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m10.82192589s) Jan 17 17:29:23.421: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m15.851558027s) Jan 17 17:29:28.450: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m20.880568846s) Jan 17 17:29:33.482: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m25.911938181s) Jan 17 17:29:38.512: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m30.942510281s) Jan 17 17:29:43.546: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m35.976483932s) Jan 17 17:29:48.579: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m41.008813819s) Jan 17 17:29:53.607: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m46.03753135s) Jan 17 17:29:58.639: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m51.069421035s) Jan 17 17:30:03.667: INFO: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 found and phase=Bound (2m56.097460418s) �[1mSTEP�[0m: Deleting sc Jan 17 17:30:08.700: FAIL: while cleanup resource Unexpected error: <errors.aggregate | len:1, cap:1>: [ [ { msg: "persistent Volume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 not deleted by dynamic provisioner: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 still exists within 3m0s", err: { s: "PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 still exists within 3m0s", }, }, ], ] persistent Volume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 not deleted by dynamic provisioner: PersistentVolume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 still exists within 3m0s occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func2() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:132 +0x20f panic({0x6c3d000, 0xc00449e540}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail.func1() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:63 +0x73 panic({0x62d5960, 0x78abec0}) /usr/local/go/src/runtime/panic.go:1038 +0x215 k8s.io/kubernetes/test/e2e/framework/ginkgowrapper.Fail({0xc00452d400, 0x26a}, {0xc002f4ed30, 0x70cbf6a, 0xc002f4ed50}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go:67 +0x1a7 k8s.io/kubernetes/test/e2e/framework.Fail({0xc00452d180, 0x255}, {0xc002448f20, 0xc00452d180, 0xc002f4ee78}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/log.go:63 +0x149 k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).match(0xc002f4ee98, {0x79beb98, 0xaa12408}, 0x0, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:79 +0x1bd k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion.(*Assertion).NotTo(0xc002f4ee98, {0x79beb98, 0xaa12408}, {0x0, 0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/onsi/gomega/internal/assertion/assertion.go:48 +0x92 k8s.io/kubernetes/test/e2e/framework.ExpectNoErrorWithOffset(0x70d7f8e, {0x78ba840, 0xc003f44640}, {0x0, 0x1, 0x2}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:46 +0xa9 k8s.io/kubernetes/test/e2e/framework.ExpectNoError(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/expect.go:40 k8s.io/kubernetes/test/e2e/storage/testsuites.createPodAndVerifyContentGid(0xc001b90c60, 0xc003cdb070, 0x1, {0x0, 0x0}, {0x0, 0x0}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:270 +0x2b5 k8s.io/kubernetes/test/e2e/storage/testsuites.(*fsGroupChangePolicyTestSuite).DefineTests.func3() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/fsgroupchangepolicy.go:233 +0x296 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc0009c81a0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [Testpattern: Dynamic PV (default fs)] fsgroupchangepolicy /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "fsgroupchangepolicy-8172". �[1mSTEP�[0m: Found 9 events. Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:25 +0000 UTC - event for ebs.csi.aws.comx4lzr: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:25 +0000 UTC - event for ebs.csi.aws.comx4lzr: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:25 +0000 UTC - event for ebs.csi.aws.comx4lzr: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "fsgroupchangepolicy-8172/ebs.csi.aws.comx4lzr" Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:28 +0000 UTC - event for ebs.csi.aws.comx4lzr: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9 Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:29 +0000 UTC - event for pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc: {default-scheduler } Scheduled: Successfully assigned fsgroupchangepolicy-8172/pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc to ip-172-20-41-188.ec2.internal Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:33 +0000 UTC - event for pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-be2f9b36-f5a8-4441-a554-d9c154b755e9" Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:34 +0000 UTC - event for pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc: {kubelet ip-172-20-41-188.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" already present on machine Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:35 +0000 UTC - event for pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc: {kubelet ip-172-20-41-188.ec2.internal} Created: Created container write-pod Jan 17 17:30:08.731: INFO: At 2023-01-17 17:26:35 +0000 UTC - event for pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc: {kubelet ip-172-20-41-188.ec2.internal} Started: Started container write-pod Jan 17 17:30:08.759: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:30:08.759: INFO: pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc ip-172-20-41-188.ec2.internal Running [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:26:29 +0000 UTC } {Ready True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:26:36 +0000 UTC } {ContainersReady True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:26:36 +0000 UTC } {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:26:29 +0000 UTC }] Jan 17 17:30:08.759: INFO: Jan 17 17:30:38.789: INFO: Unable to fetch fsgroupchangepolicy-8172/pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc/write-pod logs: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-c8b42331-2cac-4723-9fbc-eb4b14ec6afc) Jan 17 17:30:38.818: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:30:38.847: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 8003 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-mock-csi-mock-volumes-3354":"csi-mock-csi-mock-volumes-3354","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:30:03 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:03 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:03 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:03 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:30:03 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:30:38.848: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:30:38.878: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:31:08.908: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:31:08.908: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:31:08.937: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 7979 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:51 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:25:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:02 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:02 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:30:02 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:30:02 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0a02edbb51388a14d,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0114a4b7fc8ca007e,DevicePath:,},},Config:nil,},} Jan 17 17:31:08.937: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:31:08.967: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:31:38.998: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:31:38.998: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:31:39.026: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 8807 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-provisioning-4391":"ip-172-20-41-188.ec2.internal","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:35 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:27:11 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:31:25 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:31:25 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:31:25 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:31:25 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0c6f86666abaacbe2],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0c6f86666abaacbe2,DevicePath:,},},Config:nil,},} Jan 17 17:31:39.027: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:31:39.057: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:32:09.086: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:32:09.086: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:32:09.114: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 7366 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kubelet Update v1 2023-01-17 17:29:00 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:29:10 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:29:10 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:29:10 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:29:10 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:32:09.115: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:32:09.146: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:32:39.174: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:32:39.174: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:32:39.203: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 9382 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:32:03 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:32:39.203: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:32:39.234: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:32:39.292: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:32:39.293: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:32:39.293: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:32:39.293: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:32:39.293: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:32:39.293: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:32:39.293: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:32:39.293: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:32:39.293: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:32:39.293: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:32:39.293: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:32:39.293: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:32:39.293: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:32:39.293: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:32:39.293: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:32:39.293: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:32:39.403: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:32:39.403: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "fsgroupchangepolicy-8172" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\ssubPath\sshould\ssupport\sexisting\sdirectory$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:205 Jan 17 17:55:39.848: Unexpected error: <*errors.errorString | 0xc000933fd0>: { s: "expected pod \"pod-subpath-test-dynamicpv-jcx8\" success: Gave up after waiting 5m0s for pod \"pod-subpath-test-dynamicpv-jcx8\" to be \"Succeeded or Failed\"", } expected pod "pod-subpath-test-dynamicpv-jcx8" success: Gave up after waiting 5m0s for pod "pod-subpath-test-dynamicpv-jcx8" to be "Succeeded or Failed" occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770from junit_23.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:50:34.525: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename provisioning �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should support existing directory /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:205 Jan 17 17:50:34.745: INFO: Creating resource for dynamic PV Jan 17 17:50:34.745: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass provisioning-981-e2e-sc6mrjm �[1mSTEP�[0m: creating a claim Jan 17 17:50:34.778: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating pod pod-subpath-test-dynamicpv-jcx8 �[1mSTEP�[0m: Creating a pod to test subpath Jan 17 17:50:34.876: INFO: Waiting up to 5m0s for pod "pod-subpath-test-dynamicpv-jcx8" in namespace "provisioning-981" to be "Succeeded or Failed" Jan 17 17:50:34.908: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 31.089174ms Jan 17 17:50:36.940: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2.063563642s Jan 17 17:50:38.973: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4.096381912s Jan 17 17:50:41.004: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 6.127843866s Jan 17 17:50:43.037: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 8.160177949s Jan 17 17:50:45.068: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 10.191833824s Jan 17 17:50:47.101: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 12.224435327s Jan 17 17:50:49.133: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 14.256499161s Jan 17 17:50:51.165: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 16.288113498s Jan 17 17:50:53.197: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 18.32080265s Jan 17 17:50:55.230: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 20.353152099s Jan 17 17:50:57.261: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 22.384782126s Jan 17 17:50:59.294: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 24.417384454s Jan 17 17:51:01.326: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 26.449548495s Jan 17 17:51:03.359: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 28.482179084s Jan 17 17:51:05.391: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 30.514337619s Jan 17 17:51:07.422: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 32.545840075s Jan 17 17:51:09.454: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 34.577908026s Jan 17 17:51:11.486: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 36.609930567s Jan 17 17:51:13.519: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 38.642817626s Jan 17 17:51:15.552: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 40.675127254s Jan 17 17:51:17.583: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 42.706846575s Jan 17 17:51:19.615: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 44.738937397s Jan 17 17:51:21.647: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 46.770593959s Jan 17 17:51:23.680: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 48.8030769s Jan 17 17:51:25.711: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 50.834670383s Jan 17 17:51:27.743: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 52.866492085s Jan 17 17:51:29.776: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 54.899217974s Jan 17 17:51:31.807: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 56.930634835s Jan 17 17:51:33.839: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 58.96302662s Jan 17 17:51:35.871: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m0.994892385s Jan 17 17:51:37.903: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m3.026595208s Jan 17 17:51:39.935: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m5.058196208s Jan 17 17:51:41.969: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m7.093015725s Jan 17 17:51:44.001: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m9.124759285s Jan 17 17:51:46.034: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m11.157110515s Jan 17 17:51:48.066: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m13.189735163s Jan 17 17:51:50.098: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m15.22175593s Jan 17 17:51:52.130: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m17.253414144s Jan 17 17:51:54.163: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m19.286166492s Jan 17 17:51:56.195: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m21.318227991s Jan 17 17:51:58.227: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m23.350665371s Jan 17 17:52:00.259: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m25.382512199s Jan 17 17:52:02.291: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m27.414301s Jan 17 17:52:04.323: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m29.446397128s Jan 17 17:52:06.355: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m31.478081134s Jan 17 17:52:08.388: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m33.51132014s Jan 17 17:52:10.420: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m35.543293701s Jan 17 17:52:12.453: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m37.576362026s Jan 17 17:52:14.485: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m39.60811559s Jan 17 17:52:16.516: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m41.639699064s Jan 17 17:52:18.549: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m43.672178174s Jan 17 17:52:20.580: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m45.703560765s Jan 17 17:52:22.618: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m47.741198008s Jan 17 17:52:24.649: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m49.772583361s Jan 17 17:52:26.681: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m51.804948142s Jan 17 17:52:28.713: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m53.836409334s Jan 17 17:52:30.744: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m55.867845325s Jan 17 17:52:32.776: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m57.8997173s Jan 17 17:52:34.808: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 1m59.931926033s Jan 17 17:52:36.840: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m1.963942034s Jan 17 17:52:38.872: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m3.995504725s Jan 17 17:52:40.903: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m6.026837122s Jan 17 17:52:42.937: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m8.060857442s Jan 17 17:52:44.969: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m10.092543547s Jan 17 17:52:47.003: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m12.126471455s Jan 17 17:52:49.034: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m14.15800875s Jan 17 17:52:51.066: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m16.1892737s Jan 17 17:52:53.098: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m18.221730662s Jan 17 17:52:55.130: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m20.253867399s Jan 17 17:52:57.162: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m22.285102583s Jan 17 17:52:59.194: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m24.317428427s Jan 17 17:53:01.226: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m26.349944804s Jan 17 17:53:03.259: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m28.382894638s Jan 17 17:53:05.291: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m30.414444021s Jan 17 17:53:07.323: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m32.446462115s Jan 17 17:53:09.355: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m34.478780906s Jan 17 17:53:11.389: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m36.51282704s Jan 17 17:53:13.422: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m38.545270963s Jan 17 17:53:15.454: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m40.577873057s Jan 17 17:53:17.515: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m42.638301161s Jan 17 17:53:19.551: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m44.674555003s Jan 17 17:53:21.582: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m46.705938953s Jan 17 17:53:23.614: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m48.737360773s Jan 17 17:53:25.645: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m50.76870558s Jan 17 17:53:27.677: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m52.800551003s Jan 17 17:53:29.708: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m54.831935811s Jan 17 17:53:31.740: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m56.863226322s Jan 17 17:53:33.772: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 2m58.895617069s Jan 17 17:53:35.804: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m0.927530488s Jan 17 17:53:37.839: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m2.962817288s Jan 17 17:53:39.872: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m4.995060358s Jan 17 17:53:41.903: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m7.026682118s Jan 17 17:53:43.936: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m9.059448377s Jan 17 17:53:45.970: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m11.09307017s Jan 17 17:53:48.002: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m13.125754197s Jan 17 17:53:50.034: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m15.15713354s Jan 17 17:53:52.065: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m17.188570428s Jan 17 17:53:54.096: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m19.219906021s Jan 17 17:53:56.128: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m21.251346799s Jan 17 17:53:58.160: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m23.283565303s Jan 17 17:54:00.192: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m25.315409411s Jan 17 17:54:02.224: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m27.347936595s Jan 17 17:54:04.254: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m29.377419346s Jan 17 17:54:06.283: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m31.406201767s Jan 17 17:54:08.311: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m33.434746027s Jan 17 17:54:10.340: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m35.464015917s Jan 17 17:54:12.369: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m37.492818387s Jan 17 17:54:14.398: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m39.52134957s Jan 17 17:54:16.427: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m41.550882523s Jan 17 17:54:18.457: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m43.580825688s Jan 17 17:54:20.486: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m45.609907861s Jan 17 17:54:22.516: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m47.639451559s Jan 17 17:54:24.544: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m49.66795068s Jan 17 17:54:26.574: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m51.697239222s Jan 17 17:54:28.603: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m53.726411619s Jan 17 17:54:30.632: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m55.755436494s Jan 17 17:54:32.662: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m57.785409081s Jan 17 17:54:34.692: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 3m59.815228525s Jan 17 17:54:36.721: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m1.844093139s Jan 17 17:54:38.750: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m3.87317944s Jan 17 17:54:40.779: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m5.902078888s Jan 17 17:54:42.808: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m7.931666764s Jan 17 17:54:44.838: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m9.961123546s Jan 17 17:54:46.866: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m11.989720126s Jan 17 17:54:48.895: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m14.018436379s Jan 17 17:54:50.924: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m16.047108857s Jan 17 17:54:52.953: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m18.076754788s Jan 17 17:54:54.982: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m20.105861041s Jan 17 17:54:57.011: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m22.134559652s Jan 17 17:54:59.040: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m24.163435587s Jan 17 17:55:01.069: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m26.192541026s Jan 17 17:55:03.107: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m28.230825256s Jan 17 17:55:05.137: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m30.260386312s Jan 17 17:55:07.167: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m32.290345802s Jan 17 17:55:09.196: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m34.319412678s Jan 17 17:55:11.225: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m36.348410092s Jan 17 17:55:13.255: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m38.378202968s Jan 17 17:55:15.283: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m40.406863677s Jan 17 17:55:17.313: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m42.436719825s Jan 17 17:55:19.343: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m44.466326428s Jan 17 17:55:21.372: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m46.495075731s Jan 17 17:55:23.402: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m48.525231835s Jan 17 17:55:25.431: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m50.554616475s Jan 17 17:55:27.461: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m52.584508419s Jan 17 17:55:29.490: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m54.61374875s Jan 17 17:55:31.520: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m56.643872582s Jan 17 17:55:33.555: INFO: Pod "pod-subpath-test-dynamicpv-jcx8": Phase="Pending", Reason="", readiness=false. Elapsed: 4m58.6785992s Jan 17 17:55:35.617: INFO: Failed to get logs from node "ip-172-20-34-144.ec2.internal" pod "pod-subpath-test-dynamicpv-jcx8" container "init-volume-dynamicpv-jcx8": the server rejected our request for an unknown reason (get pods pod-subpath-test-dynamicpv-jcx8) Jan 17 17:55:35.649: INFO: Failed to get logs from node "ip-172-20-34-144.ec2.internal" pod "pod-subpath-test-dynamicpv-jcx8" container "test-init-subpath-dynamicpv-jcx8": the server rejected our request for an unknown reason (get pods pod-subpath-test-dynamicpv-jcx8) Jan 17 17:55:35.697: INFO: Failed to get logs from node "ip-172-20-34-144.ec2.internal" pod "pod-subpath-test-dynamicpv-jcx8" container "test-container-subpath-dynamicpv-jcx8": the server rejected our request for an unknown reason (get pods pod-subpath-test-dynamicpv-jcx8) Jan 17 17:55:35.742: INFO: Failed to get logs from node "ip-172-20-34-144.ec2.internal" pod "pod-subpath-test-dynamicpv-jcx8" container "test-container-volume-dynamicpv-jcx8": the server rejected our request for an unknown reason (get pods pod-subpath-test-dynamicpv-jcx8) �[1mSTEP�[0m: delete the pod Jan 17 17:55:35.779: INFO: Waiting for pod pod-subpath-test-dynamicpv-jcx8 to disappear Jan 17 17:55:35.807: INFO: Pod pod-subpath-test-dynamicpv-jcx8 still exists Jan 17 17:55:37.807: INFO: Waiting for pod pod-subpath-test-dynamicpv-jcx8 to disappear Jan 17 17:55:37.836: INFO: Pod pod-subpath-test-dynamicpv-jcx8 still exists Jan 17 17:55:39.807: INFO: Waiting for pod pod-subpath-test-dynamicpv-jcx8 to disappear Jan 17 17:55:39.848: INFO: Pod pod-subpath-test-dynamicpv-jcx8 no longer exists Jan 17 17:55:39.848: FAIL: Unexpected error: <*errors.errorString | 0xc000933fd0>: { s: "expected pod \"pod-subpath-test-dynamicpv-jcx8\" success: Gave up after waiting 5m0s for pod \"pod-subpath-test-dynamicpv-jcx8\" to be \"Succeeded or Failed\"", } expected pod "pod-subpath-test-dynamicpv-jcx8" success: Gave up after waiting 5m0s for pod "pod-subpath-test-dynamicpv-jcx8" to be "Succeeded or Failed" occurred Full Stack Trace k8s.io/kubernetes/test/e2e/framework.(*Framework).testContainerOutputMatcher(0xc002562800, {0x70d5075, 0x0}, 0xc002562800, 0x1, {0xc00565b120, 0x1, 0x1}, 0x22) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770 +0x176 k8s.io/kubernetes/test/e2e/framework.(*Framework).TestContainerOutput(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:567 k8s.io/kubernetes/test/e2e/storage/testsuites.testReadFile(0xc0020ecdc0, {0xc002a9a390, 0x27}, 0xc002562800, 0x1) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:691 +0x15c k8s.io/kubernetes/test/e2e/storage/testsuites.(*subPathTestSuite).DefineTests.func4() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:216 +0x1b0 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000d00820, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pod Jan 17 17:55:39.849: INFO: Deleting pod "pod-subpath-test-dynamicpv-jcx8" in namespace "provisioning-981" �[1mSTEP�[0m: Deleting pvc Jan 17 17:55:39.934: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comjzvlm" Jan 17 17:55:39.963: INFO: Waiting up to 3m0s for PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 to get deleted Jan 17 17:55:39.995: INFO: PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 found and phase=Released (31.65234ms) Jan 17 17:55:45.024: INFO: PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 found and phase=Released (5.060529994s) Jan 17 17:55:50.053: INFO: PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 found and phase=Released (10.089266077s) Jan 17 17:55:55.083: INFO: PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 found and phase=Released (15.119510553s) Jan 17 17:56:00.111: INFO: PersistentVolume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 was removed �[1mSTEP�[0m: Deleting sc [AfterEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "provisioning-981". �[1mSTEP�[0m: Found 17 events. Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:34 +0000 UTC - event for ebs.csi.aws.comjzvlm: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:34 +0000 UTC - event for ebs.csi.aws.comjzvlm: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "provisioning-981/ebs.csi.aws.comjzvlm" Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:34 +0000 UTC - event for ebs.csi.aws.comjzvlm: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:41 +0000 UTC - event for ebs.csi.aws.comjzvlm: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5 Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:41 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {default-scheduler } Scheduled: Successfully assigned provisioning-981/pod-subpath-test-dynamicpv-jcx8 to ip-172-20-34-144.ec2.internal Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:44 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-a8bc33f9-8760-401d-aaa3-524927aed3d5" Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:48 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "5439838ecc0019db7b283a4bd0b9b27be1628ec3fc24812297c28d1361b286ac" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:49 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:50 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "16e20458149f5d52c019e5f3ca6323876bb0214abd582dae12263e97ede15cac" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:52 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "938cecee3c4e71f66c2d6fd2388e1a0908f7145e2f6ee7a366d0ceb156807f76" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:54 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "3dfa8602c8105054dd89ce5c559f2d210eb1db678137c490aa61d23fbc9fda51" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:56 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "217be07a4b83a77f252a07168aede9fb068c6f3cb01e74395a58fec406fe9ea5" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:50:58 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "1abc61faf4cf89e784f5ea6bbb289c04ddf0a06f73624c0e4ca086d96e08157f" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:51:00 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d05cc770948358ad86872833270130a31edb3af66b8f51a18fd3e45e5368e084" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:51:02 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "89f1dd76773e20ae24e580f796da96d2a0832efdd22b08d669573bd2cf253143" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:51:03 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d3e0517106f34547705a2714a1e9f4312194349787da78bd23034cefa4e9be8e" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.180: INFO: At 2023-01-17 17:51:06 +0000 UTC - event for pod-subpath-test-dynamicpv-jcx8: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "373974b1251774f4fd11f3af9929a77e41b45a3733717e2fd80e1fb8c64e7e4c" network for pod "pod-subpath-test-dynamicpv-jcx8": networkPlugin cni failed to set up pod "pod-subpath-test-dynamicpv-jcx8_provisioning-981" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:00.208: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:56:00.208: INFO: Jan 17 17:56:00.237: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:56:00.266: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 22035 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-409":"ip-172-20-32-111.ec2.internal","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:41:06 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:41:10 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:26 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:26 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:26 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:51:26 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-409^1c19cc4b-968e-11ed-ac88-ee28a2623c2f],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-409^1c19cc4b-968e-11ed-ac88-ee28a2623c2f,DevicePath:,},},Config:nil,},} Jan 17 17:56:00.266: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:56:00.298: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:56:00.341: INFO: ebs-csi-node-gwd6x started at 2023-01-17 17:18:33 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:00.341: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dqfpz started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dg2zx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: inline-volume-tester-gfh6m started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-425wd started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: rs-rftvb started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-f9ttq started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-knqkm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: false, restart count 0 Jan 17 17:56:00.341: INFO: cilium-gpjwr started at 2023-01-17 17:18:33 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-765q2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-htzrq started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: rs-2ckph started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.341: INFO: coredns-autoscaler-557ccb4c66-8dwvg started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container autoscaler ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qzv5f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8bzcn started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.341: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.341: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:40:58 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:00.341: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:00.341: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:00.342: INFO: inline-volume-tester-ck866 started at 2023-01-17 17:48:55 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.342: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:00.342: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:48:55 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:00.342: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:00.342: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:00.342: INFO: coredns-867df8f45c-lkvr4 started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.342: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:00.342: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-45mmg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.342: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.565: INFO: Latency metrics for node ip-172-20-32-111.ec2.internal Jan 17 17:56:00.565: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:56:00.594: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 23661 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-6429":"ip-172-20-34-144.ec2.internal","ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:46:47 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:46:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[gcr.io/authenticated-image-pulling/alpine@sha256:7ff177862cb50c602bfe81f805969412e619c054a2bbead977d0c276988aa4a0 gcr.io/authenticated-image-pulling/alpine:3.7],SizeBytes:4206620,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854,DevicePath:,},},Config:nil,},} Jan 17 17:56:00.594: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:56:00.629: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:56:00.693: INFO: pod-6c18bab5-4f83-4e61-b379-2e810ad306d3 started at 2023-01-17 17:51:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:00.693: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:46:39 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:00.693: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:00.693: INFO: pod-subpath-test-inlinevolume-jc2d started at 2023-01-17 17:52:02 +0000 UTC (2+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Init container init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Init container test-init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container test-container-subpath-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:00.693: INFO: implicit-root-uid started at 2023-01-17 17:52:21 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container implicit-root-uid ready: false, restart count 0 Jan 17 17:56:00.693: INFO: failed-jobs-history-limit-27899633-qmt6p started at 2023-01-17 17:53:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.693: INFO: cilium-hjz7z started at 2023-01-17 17:18:41 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-z495w started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-224pp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p9sdx started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: failed-jobs-history-limit-27899632-sr42b started at 2023-01-17 17:52:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.693: INFO: rs-bv7lr started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:00.693: INFO: metadata-volume-5436ede9-13e1-49b1-8014-3f8ecacbf16d started at 2023-01-17 17:55:36 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container client-container ready: false, restart count 0 Jan 17 17:56:00.693: INFO: ebs-csi-node-zk8jc started at 2023-01-17 17:18:41 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:00.693: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:00.693: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:00.693: INFO: inline-volume-tester-jj9c4 started at 2023-01-17 17:46:47 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-4xr2s started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-5fcsm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:52:25 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:00.693: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:00.693: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-s4n98 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: rs-v564d started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.693: INFO: failed-jobs-history-limit-27899636-nglxh started at 2023-01-17 17:56:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.693: INFO: test-deployment-rpdxz-764bc7c4b7-r7b6t started at 2023-01-17 17:52:24 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:00.693: INFO: rs-b4gfv started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.693: INFO: httpd started at 2023-01-17 17:52:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:00.693: INFO: indexed-job-1-9czmc started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rxl7f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qxx72 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dpktd started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.693: INFO: hostexec-ip-172-20-34-144.ec2.internal-5nqbq started at 2023-01-17 17:51:26 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:00.693: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-r2qf2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.693: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.861: INFO: Latency metrics for node ip-172-20-34-144.ec2.internal Jan 17 17:56:00.861: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:56:00.890: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 22544 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-mock-csi-mock-volumes-1369":"csi-mock-csi-mock-volumes-1369","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:41:03 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status} {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:10 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:10 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:10 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:52:10 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-mock-csi-mock-volumes-1369^4 kubernetes.io/csi/ebs.csi.aws.com^vol-08e9c7eaaf7da4678],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-08e9c7eaaf7da4678,DevicePath:,},},Config:nil,},} Jan 17 17:56:00.890: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:56:00.921: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:56:00.960: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p4fb7 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.960: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.960: INFO: pod-subpath-test-inlinevolume-rnc8 started at 2023-01-17 17:51:31 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:00.960: INFO: Init container init-volume-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:00.960: INFO: Container test-container-subpath-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:00.960: INFO: security-context-79bf1a75-efe1-4735-b759-4b38a866b933 started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.960: INFO: Container test-container ready: false, restart count 0 Jan 17 17:56:00.960: INFO: ebs-csi-node-pgms2 started at 2023-01-17 17:18:37 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:00.960: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:00.960: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:00.960: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:00.960: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-72zcp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.960: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.960: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tnnhc started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.960: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: rs-jsbdw started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:00.961: INFO: csi-mockplugin-0 started at 2023-01-17 17:51:42 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:00.961: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:00.961: INFO: Container driver-registrar ready: true, restart count 0 Jan 17 17:56:00.961: INFO: Container mock ready: true, restart count 0 Jan 17 17:56:00.961: INFO: test-ss-0 started at 2023-01-17 17:46:54 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container webserver ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8wms8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: fail-once-non-local-2vhsb started at 2023-01-17 17:41:10 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: indexed-job-0-tfjcw started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: rs-bnfrh started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.961: INFO: failed-jobs-history-limit-27899635-c5wgb started at 2023-01-17 17:55:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: pvc-volume-tester-bg68c started at 2023-01-17 17:52:01 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container volume-tester ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-bfhwg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: inline-volume-tester-s755n started at 2023-01-17 17:41:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:00.961: INFO: rs-jqsw7 started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:00.961: INFO: pod-submit-remove-a284d838-09fe-4289-9c01-b30c15621bef started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container nginx ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cilium-6g78v started at 2023-01-17 17:18:37 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:00.961: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-kpz6g started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rcrmc started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: pod-subpath-test-inlinevolume-6dzs started at 2023-01-17 17:51:29 +0000 UTC (2+2 container statuses recorded) Jan 17 17:56:00.961: INFO: Init container init-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:00.961: INFO: Init container test-init-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:00.961: INFO: Container test-container-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:00.961: INFO: Container test-container-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:00.961: INFO: fail-once-non-local-54g2f started at 2023-01-17 17:46:56 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: fail-once-non-local-8zqc5 started at 2023-01-17 17:48:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: csi-mockplugin-attacher-0 started at 2023-01-17 17:51:42 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-cldh2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: fail-once-non-local-f79cb started at 2023-01-17 17:41:10 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-6fx7k started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:00.961: INFO: failed-jobs-history-limit-27899634-tgtr5 started at 2023-01-17 17:54:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container c ready: false, restart count 0 Jan 17 17:56:00.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qx2n9 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:00.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.314: INFO: Latency metrics for node ip-172-20-41-188.ec2.internal Jan 17 17:56:01.314: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:56:01.342: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 22019 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-48-3.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:37 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:40:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status} {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} }]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:01.343: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:56:01.373: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:56:01.411: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-ndff8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.411: INFO: rs-ckrvx started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:01.411: INFO: hostexec-ip-172-20-48-3.ec2.internal-rv92s started at 2023-01-17 17:52:38 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:01.411: INFO: pod-4543964c-3ff6-45a4-a9df-fe5b24ebd2f5 started at 2023-01-17 17:55:09 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:01.411: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qp6dg started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.411: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c9tnj started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.411: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-92l44 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.411: INFO: ebs-csi-node-jt9zf started at 2023-01-17 17:18:35 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:01.411: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:01.411: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:01.411: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:01.411: INFO: coredns-867df8f45c-spdt9 started at 2023-01-17 17:19:44 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.411: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:01.411: INFO: pod-subpath-test-preprovisionedpv-tzxp started at 2023-01-17 17:52:51 +0000 UTC (1+2 container statuses recorded) Jan 17 17:56:01.411: INFO: Init container test-init-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:01.411: INFO: Container test-container-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:01.412: INFO: Container test-container-volume-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:01.412: INFO: hostexec-ip-172-20-48-3.ec2.internal-5rgz7 started at 2023-01-17 17:55:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tdjpx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cilium-4zwpz started at 2023-01-17 17:18:35 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:01.412: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c8b47 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-82r7j started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-km28p started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-gkvnr started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.412: INFO: rs-wpnqq started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:01.412: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-sppgb started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.412: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:01.544: INFO: Latency metrics for node ip-172-20-48-3.ec2.internal Jan 17 17:56:01.544: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:56:01.573: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 22753 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:01.573: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:56:01.604: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:56:01.638: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.638: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:56:01.638: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.638: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:56:01.638: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.638: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:56:01.638: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.638: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:56:01.638: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.638: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:56:01.638: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:56:01.639: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:01.639: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.639: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:01.639: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:01.639: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:01.639: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:56:01.639: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:56:01.639: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:01.639: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:01.639: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:01.639: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:01.639: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:01.741: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:56:01.741: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "provisioning-981" for this suite.
Find pod-subpath-test-dynamicpv-jcx8 mentions in log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(default\sfs\)\]\ssubPath\sshould\ssupport\sreadOnly\sfile\sspecified\sin\sthe\svolumeMount\s\[LinuxOnly\]$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:380 Jan 17 17:36:27.160: Unexpected error: <*errors.errorString | 0xc001f32f40>: { s: "failed to get logs from pod-subpath-test-dynamicpv-jgrj for test-container-subpath-dynamicpv-jgrj: an error on the server (\"unknown\") has prevented the request from succeeding (get pods pod-subpath-test-dynamicpv-jgrj)", } failed to get logs from pod-subpath-test-dynamicpv-jgrj for test-container-subpath-dynamicpv-jgrj: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-subpath-test-dynamicpv-jgrj) occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770from junit_13.xml
[BeforeEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:35:32.330: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename provisioning �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should support readOnly file specified in the volumeMount [LinuxOnly] /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:380 Jan 17 17:35:32.525: INFO: Creating resource for dynamic PV Jan 17 17:35:32.525: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass provisioning-6158-e2e-scxbn78 �[1mSTEP�[0m: creating a claim Jan 17 17:35:32.556: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating pod pod-subpath-test-dynamicpv-jgrj �[1mSTEP�[0m: Creating a pod to test subpath Jan 17 17:35:32.645: INFO: Waiting up to 5m0s for pod "pod-subpath-test-dynamicpv-jgrj" in namespace "provisioning-6158" to be "Succeeded or Failed" Jan 17 17:35:32.673: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 27.76289ms Jan 17 17:35:34.702: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 2.056302053s Jan 17 17:35:36.731: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 4.085486085s Jan 17 17:35:38.759: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 6.113802317s Jan 17 17:35:40.788: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 8.142704021s Jan 17 17:35:42.816: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 10.171270866s Jan 17 17:35:44.846: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 12.200569872s Jan 17 17:35:46.876: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 14.230341033s Jan 17 17:35:48.923: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 16.277614228s Jan 17 17:35:50.951: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 18.306189097s Jan 17 17:35:52.980: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 20.33492455s Jan 17 17:35:55.010: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Pending", Reason="", readiness=false. Elapsed: 22.364370565s Jan 17 17:35:57.039: INFO: Pod "pod-subpath-test-dynamicpv-jgrj": Phase="Succeeded", Reason="", readiness=false. Elapsed: 24.393849598s �[1mSTEP�[0m: Saw pod success Jan 17 17:35:57.039: INFO: Pod "pod-subpath-test-dynamicpv-jgrj" satisfied condition "Succeeded or Failed" Jan 17 17:35:57.067: INFO: Trying to get logs from node ip-172-20-32-111.ec2.internal pod pod-subpath-test-dynamicpv-jgrj container test-container-subpath-dynamicpv-jgrj: <nil> Jan 17 17:36:27.096: INFO: Failed to get logs from node "ip-172-20-32-111.ec2.internal" pod "pod-subpath-test-dynamicpv-jgrj" container "test-container-subpath-dynamicpv-jgrj". an error on the server ("unknown") has prevented the request from succeeding (get pods pod-subpath-test-dynamicpv-jgrj) �[1mSTEP�[0m: delete the pod Jan 17 17:36:27.131: INFO: Waiting for pod pod-subpath-test-dynamicpv-jgrj to disappear Jan 17 17:36:27.160: INFO: Pod pod-subpath-test-dynamicpv-jgrj no longer exists Jan 17 17:36:27.160: FAIL: Unexpected error: <*errors.errorString | 0xc001f32f40>: { s: "failed to get logs from pod-subpath-test-dynamicpv-jgrj for test-container-subpath-dynamicpv-jgrj: an error on the server (\"unknown\") has prevented the request from succeeding (get pods pod-subpath-test-dynamicpv-jgrj)", } failed to get logs from pod-subpath-test-dynamicpv-jgrj for test-container-subpath-dynamicpv-jgrj: an error on the server ("unknown") has prevented the request from succeeding (get pods pod-subpath-test-dynamicpv-jgrj) occurred Full Stack Trace k8s.io/kubernetes/test/e2e/framework.(*Framework).testContainerOutputMatcher(0xc002146000, {0x70d5075, 0x0}, 0xc002146000, 0x0, {0xc0038fb120, 0x1, 0x1}, 0x2a) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770 +0x176 k8s.io/kubernetes/test/e2e/framework.(*Framework).TestContainerOutput(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:567 k8s.io/kubernetes/test/e2e/storage/testsuites.testReadFile(0xc004270b00, {0x70e285b, 0xc}, 0xc002146000, 0x0) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:691 +0x15c k8s.io/kubernetes/test/e2e/storage/testsuites.(*subPathTestSuite).DefineTests.func17() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/subpath.go:392 +0x1f0 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000cdab60, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pod Jan 17 17:36:27.161: INFO: Deleting pod "pod-subpath-test-dynamicpv-jgrj" in namespace "provisioning-6158" �[1mSTEP�[0m: Deleting pvc Jan 17 17:36:27.243: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comkxztn" Jan 17 17:36:27.272: INFO: Waiting up to 3m0s for PersistentVolume pvc-64bb2d0a-bc89-45e4-8ba5-699f11366729 to get deleted Jan 17 17:36:27.300: INFO: PersistentVolume pvc-64bb2d0a-bc89-45e4-8ba5-699f11366729 found and phase=Released (27.383035ms) Jan 17 17:36:32.328: INFO: PersistentVolume pvc-64bb2d0a-bc89-45e4-8ba5-699f11366729 was removed �[1mSTEP�[0m: Deleting sc [AfterEach] [Testpattern: Dynamic PV (default fs)] subPath /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "provisioning-6158". �[1mSTEP�[0m: Found 15 events. Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:32 +0000 UTC - event for ebs.csi.aws.comkxztn: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:32 +0000 UTC - event for ebs.csi.aws.comkxztn: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "provisioning-6158/ebs.csi.aws.comkxztn" Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:32 +0000 UTC - event for ebs.csi.aws.comkxztn: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:39 +0000 UTC - event for ebs.csi.aws.comkxztn: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-64bb2d0a-bc89-45e4-8ba5-699f11366729 Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:39 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {default-scheduler } Scheduled: Successfully assigned provisioning-6158/pod-subpath-test-dynamicpv-jgrj to ip-172-20-32-111.ec2.internal Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:41 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-64bb2d0a-bc89-45e4-8ba5-699f11366729" Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:51 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Started: Started container init-volume-dynamicpv-jgrj Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:51 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/busybox:1.29-2" already present on machine Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:51 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Created: Created container init-volume-dynamicpv-jgrj Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:52 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/agnhost:2.39" already present on machine Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:52 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Created: Created container test-init-volume-dynamicpv-jgrj Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:52 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Started: Started container test-init-volume-dynamicpv-jgrj Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:53 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Pulled: Container image "k8s.gcr.io/e2e-test-images/agnhost:2.39" already present on machine Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:53 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Created: Created container test-container-subpath-dynamicpv-jgrj Jan 17 17:36:32.387: INFO: At 2023-01-17 17:35:53 +0000 UTC - event for pod-subpath-test-dynamicpv-jgrj: {kubelet ip-172-20-32-111.ec2.internal} Started: Started container test-container-subpath-dynamicpv-jgrj Jan 17 17:36:32.415: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:36:32.415: INFO: Jan 17 17:36:32.446: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:36:32.474: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 12179 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:36:00 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:00 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:00 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:00 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:36:00 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:36:32.474: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:36:32.505: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:37:02.533: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:37:02.533: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:37:02.561: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 12764 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:36:50 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:36:50 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:37:02.562: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:37:02.623: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:37:32.665: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:37:32.666: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:37:32.694: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 13047 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:36:52 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status} {kube-controller-manager Update v1 2023-01-17 17:37:05 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:03 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:03 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:03 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:37:03 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-08edf1c2b37b90c34],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-08edf1c2b37b90c34,DevicePath:,},},Config:nil,},} Jan 17 17:37:32.694: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:37:32.724: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:38:02.752: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:38:02.752: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:38:02.780: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 13840 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-48-3.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-6301":"ip-172-20-48-3.ec2.internal","ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:37:32 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:37:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:51 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:51 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:51 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:37:51 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-6301^a43e4953-968d-11ed-b125-bebcddb42191 kubernetes.io/csi/csi-hostpath-ephemeral-6301^a442ccdf-968d-11ed-b125-bebcddb42191],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-6301^a43e4953-968d-11ed-b125-bebcddb42191,DevicePath:,},AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-6301^a442ccdf-968d-11ed-b125-bebcddb42191,DevicePath:,},},Config:nil,},} Jan 17 17:38:02.781: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:38:02.811: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:38:32.839: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:38:32.839: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:38:32.867: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 12964 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:37:10 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:38:32.868: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:38:32.898: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:38:32.955: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.955: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:38:32.955: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.955: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:38:32.955: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:38:32.956: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:38:32.956: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:38:32.956: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:38:32.956: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:38:32.956: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:38:32.956: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:38:32.956: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:38:32.956: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:38:32.956: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:38:32.956: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:38:32.956: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:38:32.956: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:38:32.956: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:38:33.087: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:38:33.087: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "provisioning-6158" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sDynamic\sPV\s\(ext4\)\]\svolumes\sshould\sallow\sexec\sof\sfiles\son\sthe\svolume$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:196 Jan 17 17:24:11.102: Unexpected error: <*errors.errorString | 0xc00301def0>: { s: "failed to get logs from exec-volume-test-dynamicpv-jqcp for exec-container-dynamicpv-jqcp: an error on the server (\"unknown\") has prevented the request from succeeding (get pods exec-volume-test-dynamicpv-jqcp)", } failed to get logs from exec-volume-test-dynamicpv-jqcp for exec-container-dynamicpv-jqcp: an error on the server ("unknown") has prevented the request from succeeding (get pods exec-volume-test-dynamicpv-jqcp) occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770from junit_16.xml
[BeforeEach] [Testpattern: Dynamic PV (ext4)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Dynamic PV (ext4)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:23:26.357: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename volume �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should allow exec of files on the volume /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:196 Jan 17 17:23:26.558: INFO: Creating resource for dynamic PV Jan 17 17:23:26.558: INFO: Using claimSize:1Gi, test suite supported size:{ 1Mi}, driver(ebs.csi.aws.com) supported size:{ 1Mi} �[1mSTEP�[0m: creating a StorageClass volume-8777-e2e-scs8scs �[1mSTEP�[0m: creating a claim Jan 17 17:23:26.588: INFO: Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil �[1mSTEP�[0m: Creating pod exec-volume-test-dynamicpv-jqcp �[1mSTEP�[0m: Creating a pod to test exec-volume-test Jan 17 17:23:26.677: INFO: Waiting up to 5m0s for pod "exec-volume-test-dynamicpv-jqcp" in namespace "volume-8777" to be "Succeeded or Failed" Jan 17 17:23:26.705: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 28.199478ms Jan 17 17:23:28.734: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 2.057152969s Jan 17 17:23:30.763: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 4.085773368s Jan 17 17:23:32.792: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 6.114418237s Jan 17 17:23:34.820: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 8.143026578s Jan 17 17:23:36.850: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 10.172994196s Jan 17 17:23:38.881: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Pending", Reason="", readiness=false. Elapsed: 12.203432551s Jan 17 17:23:40.910: INFO: Pod "exec-volume-test-dynamicpv-jqcp": Phase="Succeeded", Reason="", readiness=false. Elapsed: 14.232906043s �[1mSTEP�[0m: Saw pod success Jan 17 17:23:40.910: INFO: Pod "exec-volume-test-dynamicpv-jqcp" satisfied condition "Succeeded or Failed" Jan 17 17:23:40.939: INFO: Trying to get logs from node ip-172-20-34-144.ec2.internal pod exec-volume-test-dynamicpv-jqcp container exec-container-dynamicpv-jqcp: <nil> Jan 17 17:24:10.968: INFO: Failed to get logs from node "ip-172-20-34-144.ec2.internal" pod "exec-volume-test-dynamicpv-jqcp" container "exec-container-dynamicpv-jqcp". an error on the server ("unknown") has prevented the request from succeeding (get pods exec-volume-test-dynamicpv-jqcp) �[1mSTEP�[0m: delete the pod Jan 17 17:24:11.054: INFO: Waiting for pod exec-volume-test-dynamicpv-jqcp to disappear Jan 17 17:24:11.101: INFO: Pod exec-volume-test-dynamicpv-jqcp no longer exists Jan 17 17:24:11.101: FAIL: Unexpected error: <*errors.errorString | 0xc00301def0>: { s: "failed to get logs from exec-volume-test-dynamicpv-jqcp for exec-container-dynamicpv-jqcp: an error on the server (\"unknown\") has prevented the request from succeeding (get pods exec-volume-test-dynamicpv-jqcp)", } failed to get logs from exec-volume-test-dynamicpv-jqcp for exec-container-dynamicpv-jqcp: an error on the server ("unknown") has prevented the request from succeeding (get pods exec-volume-test-dynamicpv-jqcp) occurred Full Stack Trace k8s.io/kubernetes/test/e2e/framework.(*Framework).testContainerOutputMatcher(0xc005167920, {0x70f719c, 0x0}, 0xc00053b000, 0x0, {0xc003d99118, 0x1, 0x1}, 0x0) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/util.go:770 +0x176 k8s.io/kubernetes/test/e2e/framework.(*Framework).TestContainerOutput(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:567 k8s.io/kubernetes/test/e2e/storage/testsuites.testScriptInPod(0xc0047c69a0, {0x70d8c53, 0x0}, 0xc002ef0c30, 0xc004047080) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:255 +0x66b k8s.io/kubernetes/test/e2e/storage/testsuites.(*volumesTestSuite).DefineTests.func4() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/volumes.go:201 +0xb1 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc0009c81a0, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a �[1mSTEP�[0m: Deleting pvc Jan 17 17:24:11.164: INFO: Deleting PersistentVolumeClaim "ebs.csi.aws.comf27gq" Jan 17 17:24:11.194: INFO: Waiting up to 3m0s for PersistentVolume pvc-18d3bcf6-f954-46f1-b0fe-9f6abbecab89 to get deleted Jan 17 17:24:11.222: INFO: PersistentVolume pvc-18d3bcf6-f954-46f1-b0fe-9f6abbecab89 found and phase=Released (27.777086ms) Jan 17 17:24:16.251: INFO: PersistentVolume pvc-18d3bcf6-f954-46f1-b0fe-9f6abbecab89 was removed �[1mSTEP�[0m: Deleting sc [AfterEach] [Testpattern: Dynamic PV (ext4)] volumes /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "volume-8777". �[1mSTEP�[0m: Found 10 events. Jan 17 17:24:16.309: INFO: At 2023-01-17 17:23:26 +0000 UTC - event for ebs.csi.aws.comf27gq: {persistentvolume-controller } WaitForFirstConsumer: waiting for first consumer to be created before binding Jan 17 17:24:16.309: INFO: At 2023-01-17 17:23:26 +0000 UTC - event for ebs.csi.aws.comf27gq: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:26 +0000 UTC - event for ebs.csi.aws.comf27gq: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "volume-8777/ebs.csi.aws.comf27gq" Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:30 +0000 UTC - event for ebs.csi.aws.comf27gq: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-18d3bcf6-f954-46f1-b0fe-9f6abbecab89 Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:30 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {default-scheduler } Scheduled: Successfully assigned volume-8777/exec-volume-test-dynamicpv-jqcp to ip-172-20-34-144.ec2.internal Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:33 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-18d3bcf6-f954-46f1-b0fe-9f6abbecab89" Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:36 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {kubelet ip-172-20-34-144.ec2.internal} Pulling: Pulling image "k8s.gcr.io/e2e-test-images/nginx:1.14-2" Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:37 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {kubelet ip-172-20-34-144.ec2.internal} Pulled: Successfully pulled image "k8s.gcr.io/e2e-test-images/nginx:1.14-2" in 1.021409816s Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:37 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {kubelet ip-172-20-34-144.ec2.internal} Created: Created container exec-container-dynamicpv-jqcp Jan 17 17:24:16.310: INFO: At 2023-01-17 17:23:37 +0000 UTC - event for exec-volume-test-dynamicpv-jqcp: {kubelet ip-172-20-34-144.ec2.internal} Started: Started container exec-container-dynamicpv-jqcp Jan 17 17:24:16.337: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:24:16.337: INFO: Jan 17 17:24:16.367: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:24:16.395: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 3973 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:19:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:41 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:22:44 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:23:56 +0000 UTC,LastTransitionTime:2023-01-17 17:19:41 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-03fdd1fbca2d05e64],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-03fdd1fbca2d05e64,DevicePath:,},},Config:nil,},} Jan 17 17:24:16.395: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:24:16.426: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:24:46.454: INFO: Unable to retrieve kubelet pods for node ip-172-20-32-111.ec2.internal: error trying to reach service: dial tcp 172.20.32.111:10250: i/o timeout Jan 17 17:24:46.454: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:24:46.483: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 3867 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:06 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:22:51 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:22:54 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:23:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:23:45 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0cc170227dc73d69e],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0cc170227dc73d69e,DevicePath:,},},Config:nil,},} Jan 17 17:24:46.484: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:24:46.514: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:25:16.543: INFO: Unable to retrieve kubelet pods for node ip-172-20-34-144.ec2.internal: error trying to reach service: dial tcp 172.20.34.144:10250: i/o timeout Jan 17 17:25:16.543: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:25:16.572: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 4396 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-mock-csi-mock-volumes-5450":"csi-mock-csi-mock-volumes-5450","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:19:16 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:22:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status} {kube-controller-manager Update v1 2023-01-17 17:22:35 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:57 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:57 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:57 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:24:57 +0000 UTC,LastTransitionTime:2023-01-17 17:20:01 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-04ab1ed8f9d623bd6],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-04ab1ed8f9d623bd6,DevicePath:,},},Config:nil,},} Jan 17 17:25:16.572: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:25:16.603: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:25:46.632: INFO: Unable to retrieve kubelet pods for node ip-172-20-41-188.ec2.internal: error trying to reach service: dial tcp 172.20.41.188:10250: i/o timeout Jan 17 17:25:46.632: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:25:46.660: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 4095 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:22:37 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:22:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:14 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:14 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:24:14 +0000 UTC,LastTransitionTime:2023-01-17 17:18:34 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:24:14 +0000 UTC,LastTransitionTime:2023-01-17 17:18:54 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/ebs.csi.aws.com^vol-0ab9adf04b2689417],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/ebs.csi.aws.com^vol-0ab9adf04b2689417,DevicePath:,},},Config:nil,},} Jan 17 17:25:46.661: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:25:46.691: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:26:16.720: INFO: Unable to retrieve kubelet pods for node ip-172-20-48-3.ec2.internal: error trying to reach service: dial tcp 172.20.48.3:10250: i/o timeout Jan 17 17:26:16.720: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:26:16.749: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 2041 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:21:51 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:21:51 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:21:51 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:21:51 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:26:16.749: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:26:16.780: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:26:16.841: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:26:16.841: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:26:16.841: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:26:16.841: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:26:16.841: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:26:16.841: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:26:16.841: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:26:16.841: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:26:16.841: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:26:16.841: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:26:16.841: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:26:16.841: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:26:16.841: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:26:16.841: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:26:16.841: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:26:16.841: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:26:16.954: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:26:16.954: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "volume-8777" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\sExternal\sStorage\s\[Driver\:\sebs\.csi\.aws\.com\]\s\[Testpattern\:\sGeneric\sEphemeral\-volume\s\(default\sfs\)\s\(late\-binding\)\]\sephemeral\sshould\ssupport\stwo\spods\swhich\shave\sthe\ssame\svolume\sdefinition$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/ephemeral.go:214 Jan 17 17:55:55.585: waiting for pod with inline volume Unexpected error: <*errors.errorString | 0xc0002c82c0>: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/ephemeral.go:349from junit_09.xml
[BeforeEach] [Testpattern: Generic Ephemeral-volume (default fs) (late-binding)] ephemeral /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/framework/testsuite.go:51 [BeforeEach] [Testpattern: Generic Ephemeral-volume (default fs) (late-binding)] ephemeral /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:40:54.874: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename ephemeral �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should support two pods which have the same volume definition /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/ephemeral.go:214 Jan 17 17:40:55.321: INFO: Pod inline-volume-k2smg has the following logs: Jan 17 17:40:55.352: INFO: Deleting pod "inline-volume-k2smg" in namespace "ephemeral-1816" Jan 17 17:40:55.385: INFO: Wait up to 5m0s for pod "inline-volume-k2smg" to be fully deleted Jan 17 17:40:55.416: INFO: Creating resource for dynamic PV Jan 17 17:40:55.416: INFO: Using claimSize:1Gi, test suite supported size:{ }, driver(ebs.csi.aws.com) supported size:{ } �[1mSTEP�[0m: creating a StorageClass ephemeral-1816-e2e-scbl99x �[1mSTEP�[0m: checking the requested inline volume exists in the pod running on node {Name: Selector:map[] Affinity:nil} Jan 17 17:55:55.585: FAIL: waiting for pod with inline volume Unexpected error: <*errors.errorString | 0xc0002c82c0>: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Full Stack Trace k8s.io/kubernetes/test/e2e/storage/testsuites.EphemeralTest.TestEphemeral({{0x7b06bd0, 0xc0014aed80}, 0xc0017c3300, {0xc0041784b0, 0xe}, {0x0, 0x0}, 0xc003836000, {{0x0, 0x0}, ...}, ...}) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/ephemeral.go:349 +0x60a k8s.io/kubernetes/test/e2e/storage/testsuites.(*ephemeralTestSuite).DefineTests.func5() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/storage/testsuites/ephemeral.go:249 +0x205 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000613040, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a Jan 17 17:55:55.618: INFO: Error getting logs for pod inline-volume-tester-s755n: the server rejected our request for an unknown reason (get pods inline-volume-tester-s755n) Jan 17 17:55:55.680: INFO: Deleting pod "inline-volume-tester-s755n" in namespace "ephemeral-1816" Jan 17 17:55:55.714: INFO: Wait up to 5m0s for pod "inline-volume-tester-s755n" to be fully deleted Jan 17 17:56:05.777: INFO: Wait up to 5m0s for pod PV pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 to be fully deleted Jan 17 17:56:05.777: INFO: Waiting up to 5m0s for PersistentVolume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 to get deleted Jan 17 17:56:05.808: INFO: PersistentVolume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 found and phase=Released (30.84506ms) Jan 17 17:56:10.839: INFO: PersistentVolume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 found and phase=Released (5.062173495s) Jan 17 17:56:15.872: INFO: PersistentVolume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 found and phase=Released (10.095367567s) Jan 17 17:56:20.904: INFO: PersistentVolume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 was removed �[1mSTEP�[0m: Deleting sc [AfterEach] [Testpattern: Generic Ephemeral-volume (default fs) (late-binding)] ephemeral /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "ephemeral-1816". �[1mSTEP�[0m: Found 20 events. Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:55 +0000 UTC - event for inline-volume-k2smg: {default-scheduler } FailedScheduling: 0/5 nodes are available: 5 waiting for ephemeral volume controller to create the persistentvolumeclaim "inline-volume-k2smg-my-volume". Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:55 +0000 UTC - event for inline-volume-k2smg-my-volume: {persistentvolume-controller } ProvisioningFailed: storageclass.storage.k8s.io "no-such-storage-class" not found Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:55 +0000 UTC - event for inline-volume-tester-s755n: {default-scheduler } FailedScheduling: 0/5 nodes are available: 5 waiting for ephemeral volume controller to create the persistentvolumeclaim "inline-volume-tester-s755n-my-volume-0". Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:55 +0000 UTC - event for inline-volume-tester-s755n-my-volume-0: {persistentvolume-controller } WaitForPodScheduled: waiting for pod inline-volume-tester-s755n to be scheduled Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:56 +0000 UTC - event for inline-volume-tester-s755n-my-volume-0: {persistentvolume-controller } ExternalProvisioning: waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator Jan 17 17:56:20.968: INFO: At 2023-01-17 17:40:56 +0000 UTC - event for inline-volume-tester-s755n-my-volume-0: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } Provisioning: External provisioner is provisioning volume for claim "ephemeral-1816/inline-volume-tester-s755n-my-volume-0" Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:00 +0000 UTC - event for inline-volume-tester-s755n: {default-scheduler } Scheduled: Successfully assigned ephemeral-1816/inline-volume-tester-s755n to ip-172-20-41-188.ec2.internal Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:00 +0000 UTC - event for inline-volume-tester-s755n-my-volume-0: {ebs.csi.aws.com_ip-172-20-61-43.ec2.internal_73f712ec-3784-4655-a161-122a8633f65e } ProvisioningSucceeded: Successfully provisioned volume pvc-342168d8-7002-4b7d-81d8-9ae1808582d8 Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:05 +0000 UTC - event for inline-volume-tester-s755n: {attachdetach-controller } SuccessfulAttachVolume: AttachVolume.Attach succeeded for volume "pvc-342168d8-7002-4b7d-81d8-9ae1808582d8" Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:12 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "b084cbec751ba0268fca44d578c3df07b86b98facdc7e2bf4bfee293c8d526dc" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:14 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:15 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "a40bf39ada71471dd9020c1c832e1038fd2e6cd427af073be6bee6f71ddbef11" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:18 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "96c7d1ecfc9da9cbe854ed97c646675d16826b3987509f94ed646c4eb6e4fd20" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:22 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "4ff32f52d701acb2480cde6f97e63d8bbab809ea86112c7831720e29793cdcd2" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:24 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "5d06b7a0cb68d5214615f826e5b0fe83c081816c9784e6b4ada21521313fb623" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:26 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "b93e7c1b4567e75ab65bd28f406ee8b6b71116a594fccf3f83d9694d68ca3198" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:30 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d34884e62a297cb17dc312932d0db4624dd14717e1e3ee0fe4fdec9284337835" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:34 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "973d4a11d30f0bba57675e18b24cd32df65d135cfe13009a54b741eb50cc6755" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:35 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "8852b4aeb3118293d4a4dcb0e9082a2adc8421d9e1d0da854b58a6aa16e63882" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.968: INFO: At 2023-01-17 17:41:51 +0000 UTC - event for inline-volume-tester-s755n: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "72eca7a7ea7d6cf714b3b7b645a4ecf42cbc077d598daa499a9cc47cf946f3a0" network for pod "inline-volume-tester-s755n": networkPlugin cni failed to set up pod "inline-volume-tester-s755n_ephemeral-1816" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:20.998: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:56:20.998: INFO: Jan 17 17:56:21.030: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:56:21.062: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 23886 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-9602":"ip-172-20-32-111.ec2.internal","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} } {kubelet Update v1 2023-01-17 17:56:01 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:21.062: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:56:21.096: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:56:21.136: INFO: coredns-autoscaler-557ccb4c66-8dwvg started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container autoscaler ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qzv5f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-htzrq started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: rs-2ckph started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8bzcn started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: inline-volume-tester-ck866 started at 2023-01-17 17:48:55 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:21.136: INFO: coredns-867df8f45c-lkvr4 started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:21.136: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:48:55 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:21.136: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-45mmg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: ebs-csi-node-gwd6x started at 2023-01-17 17:18:33 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:21.136: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dqfpz started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dg2zx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-f9ttq started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-knqkm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: false, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-425wd started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.136: INFO: rs-rftvb started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:21.136: INFO: cilium-gpjwr started at 2023-01-17 17:18:33 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:21.136: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:21.136: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-765q2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.136: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.597: INFO: Latency metrics for node ip-172-20-32-111.ec2.internal Jan 17 17:56:21.598: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:56:21.629: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 23661 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-6429":"ip-172-20-34-144.ec2.internal","ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:46:47 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:46:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[gcr.io/authenticated-image-pulling/alpine@sha256:7ff177862cb50c602bfe81f805969412e619c054a2bbead977d0c276988aa4a0 gcr.io/authenticated-image-pulling/alpine:3.7],SizeBytes:4206620,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854,DevicePath:,},},Config:nil,},} Jan 17 17:56:21.630: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:56:21.663: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-224pp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p9sdx started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: failed-jobs-history-limit-27899632-sr42b started at 2023-01-17 17:52:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container c ready: false, restart count 0 Jan 17 17:56:21.706: INFO: implicit-root-uid started at 2023-01-17 17:52:21 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container implicit-root-uid ready: false, restart count 0 Jan 17 17:56:21.706: INFO: failed-jobs-history-limit-27899633-qmt6p started at 2023-01-17 17:53:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container c ready: false, restart count 0 Jan 17 17:56:21.706: INFO: cilium-hjz7z started at 2023-01-17 17:18:41 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:21.706: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-z495w started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-4xr2s started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-5fcsm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:52:25 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:21.706: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:21.706: INFO: rs-bv7lr started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:21.706: INFO: metadata-volume-5436ede9-13e1-49b1-8014-3f8ecacbf16d started at 2023-01-17 17:55:36 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container client-container ready: false, restart count 0 Jan 17 17:56:21.706: INFO: ebs-csi-node-zk8jc started at 2023-01-17 17:18:41 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:21.706: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:21.706: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:21.706: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:21.706: INFO: inline-volume-tester-jj9c4 started at 2023-01-17 17:46:47 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-s4n98 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: rs-v564d started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:21.706: INFO: failed-jobs-history-limit-27899636-nglxh started at 2023-01-17 17:56:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container c ready: false, restart count 0 Jan 17 17:56:21.706: INFO: test-deployment-rpdxz-764bc7c4b7-r7b6t started at 2023-01-17 17:52:24 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:21.706: INFO: indexed-job-1-9czmc started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container c ready: false, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rxl7f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: pod-projected-configmaps-0d852a06-00ac-419f-ae07-bad1849c14cc started at 2023-01-17 17:56:02 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container agnhost-container ready: false, restart count 0 Jan 17 17:56:21.706: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:56:13 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:21.706: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:21.706: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:21.706: INFO: rs-b4gfv started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:21.706: INFO: httpd started at 2023-01-17 17:52:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dpktd started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qxx72 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-r2qf2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:21.706: INFO: hostexec-ip-172-20-34-144.ec2.internal-5nqbq started at 2023-01-17 17:51:26 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:21.706: INFO: pod-subpath-test-inlinevolume-jc2d started at 2023-01-17 17:52:02 +0000 UTC (2+1 container statuses recorded) Jan 17 17:56:21.706: INFO: Init container init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:21.707: INFO: Init container test-init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:21.707: INFO: Container test-container-subpath-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:21.707: INFO: pod-6c18bab5-4f83-4e61-b379-2e810ad306d3 started at 2023-01-17 17:51:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:21.707: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:21.707: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:46:39 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:21.707: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:21.707: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:22.189: INFO: Latency metrics for node ip-172-20-34-144.ec2.internal Jan 17 17:56:22.189: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:56:22.221: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 23782 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-mock-csi-mock-volumes-1369":"csi-mock-csi-mock-volumes-1369","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:41:03 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status} {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} }]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-mock-csi-mock-volumes-1369^4],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:22.222: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:56:22.257: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:56:22.298: INFO: pod-subpath-test-inlinevolume-6dzs started at 2023-01-17 17:51:29 +0000 UTC (2+2 container statuses recorded) Jan 17 17:56:22.298: INFO: Init container init-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:22.298: INFO: Init container test-init-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:22.298: INFO: Container test-container-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:22.298: INFO: Container test-container-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:22.298: INFO: cilium-6g78v started at 2023-01-17 17:18:37 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:22.298: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:22.298: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-kpz6g started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.298: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rcrmc started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.298: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-cldh2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.298: INFO: fail-once-non-local-8zqc5 started at 2023-01-17 17:48:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Container c ready: false, restart count 0 Jan 17 17:56:22.298: INFO: csi-mockplugin-attacher-0 started at 2023-01-17 17:51:42 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.298: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:22.298: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-6fx7k started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: failed-jobs-history-limit-27899634-tgtr5 started at 2023-01-17 17:54:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container c ready: false, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qx2n9 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p4fb7 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: pod-subpath-test-inlinevolume-rnc8 started at 2023-01-17 17:51:31 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Init container init-volume-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:22.299: INFO: Container test-container-subpath-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:22.299: INFO: security-context-79bf1a75-efe1-4735-b759-4b38a866b933 started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container test-container ready: false, restart count 0 Jan 17 17:56:22.299: INFO: rs-jsbdw started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:22.299: INFO: csi-mockplugin-0 started at 2023-01-17 17:51:42 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:22.299: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:22.299: INFO: Container driver-registrar ready: true, restart count 0 Jan 17 17:56:22.299: INFO: Container mock ready: true, restart count 0 Jan 17 17:56:22.299: INFO: ebs-csi-node-pgms2 started at 2023-01-17 17:18:37 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:22.299: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:22.299: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:22.299: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-72zcp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tnnhc started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: indexed-job-0-tfjcw started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container c ready: false, restart count 0 Jan 17 17:56:22.299: INFO: rs-bnfrh started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:22.299: INFO: failed-jobs-history-limit-27899635-c5wgb started at 2023-01-17 17:55:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container c ready: false, restart count 0 Jan 17 17:56:22.299: INFO: test-ss-0 started at 2023-01-17 17:46:54 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container webserver ready: false, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8wms8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: fail-once-non-local-2vhsb started at 2023-01-17 17:41:10 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container c ready: false, restart count 0 Jan 17 17:56:22.299: INFO: kube-proxy-mode-detector started at 2023-01-17 17:56:16 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:22.299: INFO: pvc-volume-tester-bg68c started at 2023-01-17 17:52:01 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container volume-tester ready: false, restart count 0 Jan 17 17:56:22.299: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-bfhwg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.299: INFO: rs-jqsw7 started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:22.299: INFO: pod-submit-remove-a284d838-09fe-4289-9c01-b30c15621bef started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.299: INFO: Container nginx ready: false, restart count 0 Jan 17 17:56:22.670: INFO: Latency metrics for node ip-172-20-41-188.ec2.internal Jan 17 17:56:22.670: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:56:22.702: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 22019 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubelet_cleanup:true kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-48-3.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:37 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:40:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status} {e2e.test Update v1 2023-01-17 17:41:05 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubelet_cleanup":{}}}} }]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:22.702: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:56:22.736: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:56:22.776: INFO: cilium-4zwpz started at 2023-01-17 17:18:35 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:22.777: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c8b47 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-82r7j started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tdjpx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-gkvnr started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: rs-wpnqq started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-sppgb started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-km28p started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-ndff8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: rs-ckrvx started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:22.777: INFO: hostexec-ip-172-20-48-3.ec2.internal-rv92s started at 2023-01-17 17:52:38 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:22.777: INFO: pod-4543964c-3ff6-45a4-a9df-fe5b24ebd2f5 started at 2023-01-17 17:55:09 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c9tnj started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-92l44 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.777: INFO: ebs-csi-node-jt9zf started at 2023-01-17 17:18:35 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:22.777: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:22.777: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:22.777: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:22.777: INFO: coredns-867df8f45c-spdt9 started at 2023-01-17 17:19:44 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:22.777: INFO: pod-subpath-test-preprovisionedpv-tzxp started at 2023-01-17 17:52:51 +0000 UTC (1+2 container statuses recorded) Jan 17 17:56:22.777: INFO: Init container test-init-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:22.777: INFO: Container test-container-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:22.777: INFO: Container test-container-volume-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:22.777: INFO: hostexec-ip-172-20-48-3.ec2.internal-5rgz7 started at 2023-01-17 17:55:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:22.777: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qp6dg started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:22.777: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:22.925: INFO: Latency metrics for node ip-172-20-48-3.ec2.internal Jan 17 17:56:22.925: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:56:22.956: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 22753 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:22.956: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:56:22.990: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:56:23.026: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:56:23.026: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:56:23.026: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:56:23.026: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:56:23.026: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:23.026: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:56:23.026: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:56:23.026: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:23.026: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:23.026: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:23.026: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:56:23.026: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:56:23.026: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:23.026: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:23.026: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:23.026: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:23.137: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:56:23.138: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "ephemeral-1816" for this suite.
Find with mentions in log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\s\[sig\-apps\]\sCronJob\sshould\sdelete\sfailed\sfinished\sjobs\swith\slimit\sof\sone\sjob$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/cronjob.go:289 Jan 17 17:56:27.467: Failed to ensure a finished cronjob exists in namespace cronjob-3813 Unexpected error: <*errors.errorString | 0xc0002482c0>: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/cronjob.go:475from junit_05.xml
[BeforeEach] [sig-apps] CronJob /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:51:27.162: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename cronjob �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [It] should delete failed finished jobs with limit of one job /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/cronjob.go:289 �[1mSTEP�[0m: Creating an AllowConcurrent cronjob with custom history limit �[1mSTEP�[0m: Ensuring a finished job exists Jan 17 17:56:27.467: FAIL: Failed to ensure a finished cronjob exists in namespace cronjob-3813 Unexpected error: <*errors.errorString | 0xc0002482c0>: { s: "timed out waiting for the condition", } timed out waiting for the condition occurred Full Stack Trace k8s.io/kubernetes/test/e2e/apps.ensureHistoryLimits({0x7b06bd0, 0xc001ef7080}, {0xc002e4d6a0, 0xc}, 0x70debd5) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/cronjob.go:475 +0x1b2 k8s.io/kubernetes/test/e2e/apps.glob..func1.9() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/cronjob.go:296 +0xf6 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x243a8f9) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc000a50b60, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [sig-apps] CronJob /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:186 �[1mSTEP�[0m: Collecting events from namespace "cronjob-3813". �[1mSTEP�[0m: Found 67 events. Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:00 +0000 UTC - event for failed-jobs-history-limit: {cronjob-controller } SuccessfulCreate: Created job failed-jobs-history-limit-27899632 Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:00 +0000 UTC - event for failed-jobs-history-limit-27899632: {job-controller } SuccessfulCreate: Created pod: failed-jobs-history-limit-27899632-sr42b Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:00 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {default-scheduler } Scheduled: Successfully assigned cronjob-3813/failed-jobs-history-limit-27899632-sr42b to ip-172-20-34-144.ec2.internal Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:01 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "3b9f26c264ef0a68a9495028311666e563dc55b75507f75bf157fe6811747a07" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:04 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:06 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "5564adb0fd82d1fd5763554f08f3271e97b284e2aa6be24136eea57bde401016" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:08 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "a9c8aecb7b1a30fa866bb9aca74989dac736853b2c1a2a96e5ad04e3b80e9670" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:10 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "50c638ada35124f0e773c66b92774ffcbad870d6e3c8cc58bf4baf4a5e7eb0ce" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:12 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "2736cb9a6c1285fe8fad5368b38459c8edcb69cbbba11ee491f32b10c5e1ad52" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:15 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "10f37802c5352652d5b2c517dd7a815a1bf6f32ea0c84537ad83ee987bdfb639" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:17 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d819c1b5313875aed018f538555171179b59681916d3ceaa75bd8f93a2f014f2" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:20 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "5ecf10c6cbe414442a76e96581a7c4aee778849d053cdf8069889f3d7a2c41d9" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:22 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "874fb2c2e2eabd9d0908f364ebc511affb8c1b836ba04697fa7c56c90354a29f" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:52:24 +0000 UTC - event for failed-jobs-history-limit-27899632-sr42b: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "7833c8db60cbadd2f36a9684aeda4e68a989a159e8e2cfdd4891c81a7eeaad47" network for pod "failed-jobs-history-limit-27899632-sr42b": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899632-sr42b_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:00 +0000 UTC - event for failed-jobs-history-limit: {cronjob-controller } SuccessfulCreate: Created job failed-jobs-history-limit-27899633 Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:00 +0000 UTC - event for failed-jobs-history-limit-27899633: {job-controller } SuccessfulCreate: Created pod: failed-jobs-history-limit-27899633-qmt6p Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:00 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {default-scheduler } Scheduled: Successfully assigned cronjob-3813/failed-jobs-history-limit-27899633-qmt6p to ip-172-20-34-144.ec2.internal Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:01 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "f635cd0ff21a66f588352560cfac0e7e6494858af231786954388080b81b6115" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:03 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:04 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "61716bd96ff42cbc8f3797795a75793da52e1f775ed70df5dbe76982a2c014b9" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:06 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "3485608026d89232a37bf6accf7f5b7dda79f5eea3eef3a8100b8df86b9baafd" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:09 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d79b4216893b832a0c2d3dc237c8bb5b33faeb7f7c6a17fd94f5fdf99421f325" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:12 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "16718a190f69d47e65fbe7b3a6e009f316ba64efccda804a38192e4239146b6a" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:13 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "8d5fac87b635dad7aaab3bb63e9a6e3bc5a651963613cc4d04a8aabe61e07abd" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:16 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "6136c7655595330fa80a3f90818b8db92c66eae746a261a4874c6877b300e002" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:19 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "65491c2d2ce8d9ac2d9105607f206d0de8d223a537be0181d2baecd1cc73a2f0" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:21 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "50bd705dcc26052e5bb3130f3563b65bc547ffc2632ed86484524659919822e2" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:53:25 +0000 UTC - event for failed-jobs-history-limit-27899633-qmt6p: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "f2cae845eeca1a4a9b0cd2ae603b18540e16e0fc7b8be9511cbb98077ac03c88" network for pod "failed-jobs-history-limit-27899633-qmt6p": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899633-qmt6p_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.499: INFO: At 2023-01-17 17:54:00 +0000 UTC - event for failed-jobs-history-limit: {cronjob-controller } SuccessfulCreate: Created job failed-jobs-history-limit-27899634 Jan 17 17:56:27.499: INFO: At 2023-01-17 17:54:00 +0000 UTC - event for failed-jobs-history-limit-27899634: {job-controller } SuccessfulCreate: Created pod: failed-jobs-history-limit-27899634-tgtr5 Jan 17 17:56:27.499: INFO: At 2023-01-17 17:54:00 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {default-scheduler } Scheduled: Successfully assigned cronjob-3813/failed-jobs-history-limit-27899634-tgtr5 to ip-172-20-41-188.ec2.internal Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:01 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "285af26082595b45bc3eccd7071266c3c4ca9a73008abb44baa6540fa7568464" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:04 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:06 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "be31ea8d339735c4f4b3bc18b5f42b3b12984b804fdd2653e234b46f3a64bd2f" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:08 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "4db8de3f8ab4a78f7bb8fb4853bb1108a0d9ee8d1bc144db12ea8951c2ac01e6" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:11 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "78de7dbc7a2b3c36bd8cdcebb1a7877c0e571c9179d2e8ac8955d2964893e6be" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:17 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "ff59854c0e1278c2f54aa3e321b8bc058742aeddbe26edb43b72a448c96b0552" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:20 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "94e6371702d0d7eef66f8233c89b0faf2569fe5595b0d166dc9e1e9d11ed63c9" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:21 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "e48e6126791314adc6c10f512a8497fec9bba1a78959d3235ad15599d0b6ca57" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:24 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "4b65f1d66ed752596bd2e1c32af19a1dce8989d15a752fd381cbdb1950ac1cdc" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:27 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "1bd08817abeb433524e1fd4a95ce549268266586f0b31e24b249abdaa41b5070" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:54:30 +0000 UTC - event for failed-jobs-history-limit-27899634-tgtr5: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "c9adede8ad7de88fce0de677b08721ea23cb427e5ebdb220e4acaf874adac8ee" network for pod "failed-jobs-history-limit-27899634-tgtr5": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899634-tgtr5_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:00 +0000 UTC - event for failed-jobs-history-limit: {cronjob-controller } SuccessfulCreate: Created job failed-jobs-history-limit-27899635 Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:00 +0000 UTC - event for failed-jobs-history-limit-27899635: {job-controller } SuccessfulCreate: Created pod: failed-jobs-history-limit-27899635-c5wgb Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:00 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {default-scheduler } Scheduled: Successfully assigned cronjob-3813/failed-jobs-history-limit-27899635-c5wgb to ip-172-20-41-188.ec2.internal Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:02 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "8bf08baba15822f162636729150ce3ce03e882c6702a19f6faee2b40e84b029f" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:04 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:05 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "c58c9fb8427c02c66b0d6027d707810aaa46643e55f29e10fbaee618fd53434e" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:08 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "5d346d4fb90188d2e9031fb48695107ede4e6c77e68b2bc452ddaf63257146cf" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:11 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "aa9603b99224299490655a295f8375c7610c479716be1742cfaa7050df89b4a0" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:14 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "015869f629e00b8c50e116773b5b1c6db358c5573c5192aab235e9e53d3bb576" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:16 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "06cc28515da6f7ab00241fe868e30b72d0b1140e89531d7a655ef9b0d0e12ef1" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:20 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "2c4f0d5c883378e6def1993adaeddda33c09c755dcb3668275783bcc0c0b3736" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:23 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "0dd89c79531119d199f79b4f6bcf2208eacca5fbe634482690ef3b0982a91cd0" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:27 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "e60a19d1cda858bab1bf3845849d315e7ed9eb626f1e5adad15aab9b695a5d85" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:55:29 +0000 UTC - event for failed-jobs-history-limit-27899635-c5wgb: {kubelet ip-172-20-41-188.ec2.internal} FailedCreatePodSandBox: (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "f3d4bf5c002a1f121e2252dc4bb4978084b799ad8904f6f4221d209c3fed06a8" network for pod "failed-jobs-history-limit-27899635-c5wgb": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899635-c5wgb_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:00 +0000 UTC - event for failed-jobs-history-limit: {cronjob-controller } SuccessfulCreate: Created job failed-jobs-history-limit-27899636 Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:00 +0000 UTC - event for failed-jobs-history-limit-27899636: {job-controller } SuccessfulCreate: Created pod: failed-jobs-history-limit-27899636-nglxh Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:00 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {default-scheduler } Scheduled: Successfully assigned cronjob-3813/failed-jobs-history-limit-27899636-nglxh to ip-172-20-34-144.ec2.internal Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:02 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "afc1e3a5eea0fd7ce6032290811422b9448862d980c21c3ae75553aaf791989c" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:04 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} SandboxChanged: Pod sandbox changed, it will be killed and re-created. Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:06 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "a59c0629705d0c45f5e3086a7c1cba2d5976f77c0bc2d1b24f952cf5937fa12f" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:10 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d05b567525cc7791d944d1795565eaef3fc58fd072f1a401e5de7bcf599a2f2a" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:14 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "e66c5c769807dca6d8786c47c94fa138b2eaa1280e61cb679db5ade60970dcfc" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:16 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "9f5cabd733cc23172ea8284a525201297ec198f5ac61129df64565f4accaf06d" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:20 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "3724c7e35cc8fbd0d7c447522726674fa356be36dc6861f7630f4ae8337c2521" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.500: INFO: At 2023-01-17 17:56:24 +0000 UTC - event for failed-jobs-history-limit-27899636-nglxh: {kubelet ip-172-20-34-144.ec2.internal} FailedCreatePodSandBox: Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "d443ed772d33b9cccfef4079b225a1c7926be4911c0d31546950ae36fb6d2018" network for pod "failed-jobs-history-limit-27899636-nglxh": networkPlugin cni failed to set up pod "failed-jobs-history-limit-27899636-nglxh_cronjob-3813" network: unable to allocate IP via local cilium agent: [POST /ipam][502] postIpamFailure No more IPs available Jan 17 17:56:27.533: INFO: POD NODE PHASE GRACE CONDITIONS Jan 17 17:56:27.533: INFO: failed-jobs-history-limit-27899632-sr42b ip-172-20-34-144.ec2.internal Pending [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:52:00 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:52:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:52:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:52:00 +0000 UTC }] Jan 17 17:56:27.534: INFO: failed-jobs-history-limit-27899633-qmt6p ip-172-20-34-144.ec2.internal Pending [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:53:00 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:53:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:53:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:53:00 +0000 UTC }] Jan 17 17:56:27.534: INFO: failed-jobs-history-limit-27899634-tgtr5 ip-172-20-41-188.ec2.internal Pending [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:54:00 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:54:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:54:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:54:00 +0000 UTC }] Jan 17 17:56:27.534: INFO: failed-jobs-history-limit-27899635-c5wgb ip-172-20-41-188.ec2.internal Pending [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:55:00 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:55:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:55:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:55:00 +0000 UTC }] Jan 17 17:56:27.534: INFO: failed-jobs-history-limit-27899636-nglxh ip-172-20-34-144.ec2.internal Pending [{Initialized True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:56:00 +0000 UTC } {Ready False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:56:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {ContainersReady False 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:56:00 +0000 UTC ContainersNotReady containers with unready status: [c]} {PodScheduled True 0001-01-01 00:00:00 +0000 UTC 2023-01-17 17:56:00 +0000 UTC }] Jan 17 17:56:27.534: INFO: Jan 17 17:56:27.624: INFO: Unable to fetch cronjob-3813/failed-jobs-history-limit-27899632-sr42b/c logs: the server rejected our request for an unknown reason (get pods failed-jobs-history-limit-27899632-sr42b) Jan 17 17:56:27.672: INFO: Unable to fetch cronjob-3813/failed-jobs-history-limit-27899633-qmt6p/c logs: the server rejected our request for an unknown reason (get pods failed-jobs-history-limit-27899633-qmt6p) Jan 17 17:56:27.714: INFO: Unable to fetch cronjob-3813/failed-jobs-history-limit-27899634-tgtr5/c logs: the server rejected our request for an unknown reason (get pods failed-jobs-history-limit-27899634-tgtr5) Jan 17 17:56:27.746: INFO: Unable to fetch cronjob-3813/failed-jobs-history-limit-27899635-c5wgb/c logs: the server rejected our request for an unknown reason (get pods failed-jobs-history-limit-27899635-c5wgb) Jan 17 17:56:27.797: INFO: Unable to fetch cronjob-3813/failed-jobs-history-limit-27899636-nglxh/c logs: the server rejected our request for an unknown reason (get pods failed-jobs-history-limit-27899636-nglxh) Jan 17 17:56:27.828: INFO: Logging node info for node ip-172-20-32-111.ec2.internal Jan 17 17:56:27.859: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-32-111.ec2.internal 3554cf0a-0754-4b35-a76e-c652e97b0382 24061 0 2023-01-17 17:18:32 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-32-111.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-32-111.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-9602":"ip-172-20-32-111.ec2.internal","ebs.csi.aws.com":"i-00faba2ff01212580"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:18:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kops-controller Update v1 2023-01-17 17:18:33 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.1.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:56:01 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.1.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-00faba2ff01212580,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.1.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:56:01 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.32.111,},NodeAddress{Type:ExternalIP,Address:174.129.44.160,},NodeAddress{Type:Hostname,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-32-111.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-174-129-44-160.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec26b27643e4a7ed2caf1404408a48fb,SystemUUID:ec26b276-43e4-a7ed-2caf-1404408a48fb,BootID:98a646dd-1f9f-47e0-9bf0-17484007e806,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[registry.k8s.io/cpa/cluster-proportional-autoscaler@sha256:68d396900aeaa072c1f27289485fdac29834045a6f3ffe369bf389d830ef572d registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.6],SizeBytes:47910609,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:27.859: INFO: Logging kubelet events for node ip-172-20-32-111.ec2.internal Jan 17 17:56:27.891: INFO: Logging pods the kubelet thinks is on node ip-172-20-32-111.ec2.internal Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-45mmg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: ebs-csi-node-gwd6x started at 2023-01-17 17:18:33 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:27.930: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dqfpz started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dg2zx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-f9ttq started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-knqkm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: false, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-425wd started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: rs-rftvb started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:27.930: INFO: cilium-gpjwr started at 2023-01-17 17:18:33 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-765q2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: coredns-autoscaler-557ccb4c66-8dwvg started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container autoscaler ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qzv5f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-htzrq started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: rs-2ckph started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:27.930: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8bzcn started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:27.930: INFO: inline-volume-tester-ck866 started at 2023-01-17 17:48:55 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:27.930: INFO: coredns-867df8f45c-lkvr4 started at 2023-01-17 17:18:53 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:27.930: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:27.930: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:48:55 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:27.930: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:27.930: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:28.057: INFO: Latency metrics for node ip-172-20-32-111.ec2.internal Jan 17 17:56:28.057: INFO: Logging node info for node ip-172-20-34-144.ec2.internal Jan 17 17:56:28.088: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-34-144.ec2.internal 3805c173-34f7-471c-8fe7-ea0150cd5e85 24063 0 2023-01-17 17:18:40 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-34-144.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-34-144.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-hostpath-ephemeral-6429":"ip-172-20-34-144.ec2.internal","ebs.csi.aws.com":"i-0f7568282d9dd48c6"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:40 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:26 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.4.0/24\"":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:46:47 +0000 UTC FieldsV1 {"f:status":{"f:volumesAttached":{}}} status} {kubelet Update v1 2023-01-17 17:46:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.4.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f7568282d9dd48c6,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.4.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:55:44 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.34.144,},NodeAddress{Type:ExternalIP,Address:34.201.143.205,},NodeAddress{Type:Hostname,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-34-144.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-34-201-143-205.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2969b658d9c26b5a80456f8708bc95,SystemUUID:ec2969b6-58d9-c26b-5a80-456f8708bc95,BootID:5514abf3-0d13-41c2-835b-e86a15311ed0,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonroot@sha256:b9e2958a3dd879e3cf11142228c6d073d0fc4ea2e857c3be6f4fb0ab5fb2c937 k8s.gcr.io/e2e-test-images/nonroot:1.2],SizeBytes:42321438,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[gcr.io/authenticated-image-pulling/alpine@sha256:7ff177862cb50c602bfe81f805969412e619c054a2bbead977d0c276988aa4a0 gcr.io/authenticated-image-pulling/alpine:3.7],SizeBytes:4206620,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854],VolumesAttached:[]AttachedVolume{AttachedVolume{Name:kubernetes.io/csi/csi-hostpath-ephemeral-6429^e99edc2c-968e-11ed-800a-8238f9b3e854,DevicePath:,},},Config:nil,},} Jan 17 17:56:28.088: INFO: Logging kubelet events for node ip-172-20-34-144.ec2.internal Jan 17 17:56:28.120: INFO: Logging pods the kubelet thinks is on node ip-172-20-34-144.ec2.internal Jan 17 17:56:28.181: INFO: inline-volume-tester-jj9c4 started at 2023-01-17 17:46:47 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container csi-volume-tester ready: false, restart count 0 Jan 17 17:56:28.181: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-4xr2s started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.181: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-5fcsm started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.181: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:52:25 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:28.181: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:28.181: INFO: rs-bv7lr started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:28.181: INFO: metadata-volume-5436ede9-13e1-49b1-8014-3f8ecacbf16d started at 2023-01-17 17:55:36 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container client-container ready: false, restart count 0 Jan 17 17:56:28.181: INFO: ebs-csi-node-zk8jc started at 2023-01-17 17:18:41 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:28.181: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:28.181: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:28.181: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:28.181: INFO: rs-v564d started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:28.181: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-s4n98 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.181: INFO: test-deployment-rpdxz-764bc7c4b7-r7b6t started at 2023-01-17 17:52:24 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:28.181: INFO: failed-jobs-history-limit-27899636-nglxh started at 2023-01-17 17:56:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.181: INFO: httpd started at 2023-01-17 17:52:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container httpd ready: false, restart count 0 Jan 17 17:56:28.181: INFO: indexed-job-1-9czmc started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.181: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rxl7f started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.181: INFO: pod-projected-configmaps-0d852a06-00ac-419f-ae07-bad1849c14cc started at 2023-01-17 17:56:02 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container agnhost-container ready: false, restart count 0 Jan 17 17:56:28.181: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:56:13 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:28.181: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-provisioner ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-resizer ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container csi-snapshotter ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container hostpath ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container liveness-probe ready: false, restart count 0 Jan 17 17:56:28.181: INFO: Container node-driver-registrar ready: false, restart count 0 Jan 17 17:56:28.181: INFO: affinity-clusterip-timeout-924gz started at 2023-01-17 17:56:23 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.181: INFO: Container affinity-clusterip-timeout ready: false, restart count 0 Jan 17 17:56:28.182: INFO: rs-b4gfv started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-dpktd started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qxx72 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-r2qf2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: hostexec-ip-172-20-34-144.ec2.internal-5nqbq started at 2023-01-17 17:51:26 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:28.182: INFO: csi-hostpathplugin-0 started at 2023-01-17 17:46:39 +0000 UTC (0+7 container statuses recorded) Jan 17 17:56:28.182: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container csi-snapshotter ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container hostpath ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:28.182: INFO: pod-subpath-test-inlinevolume-cvsk started at 2023-01-17 17:56:23 +0000 UTC (2+2 container statuses recorded) Jan 17 17:56:28.182: INFO: Init container init-volume-inlinevolume-cvsk ready: false, restart count 0 Jan 17 17:56:28.182: INFO: Init container test-init-subpath-inlinevolume-cvsk ready: false, restart count 0 Jan 17 17:56:28.182: INFO: Container test-container-subpath-inlinevolume-cvsk ready: false, restart count 0 Jan 17 17:56:28.182: INFO: Container test-container-volume-inlinevolume-cvsk ready: false, restart count 0 Jan 17 17:56:28.182: INFO: pod-subpath-test-inlinevolume-jc2d started at 2023-01-17 17:52:02 +0000 UTC (2+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Init container init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:28.182: INFO: Init container test-init-volume-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:28.182: INFO: Container test-container-subpath-inlinevolume-jc2d ready: false, restart count 0 Jan 17 17:56:28.182: INFO: pod-6c18bab5-4f83-4e61-b379-2e810ad306d3 started at 2023-01-17 17:51:30 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-z495w started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-224pp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p9sdx started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.182: INFO: failed-jobs-history-limit-27899632-sr42b started at 2023-01-17 17:52:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.182: INFO: implicit-root-uid started at 2023-01-17 17:52:21 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container implicit-root-uid ready: false, restart count 0 Jan 17 17:56:28.182: INFO: failed-jobs-history-limit-27899633-qmt6p started at 2023-01-17 17:53:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.182: INFO: cilium-hjz7z started at 2023-01-17 17:18:41 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:28.182: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:28.182: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:28.523: INFO: Latency metrics for node ip-172-20-34-144.ec2.internal Jan 17 17:56:28.523: INFO: Logging node info for node ip-172-20-41-188.ec2.internal Jan 17 17:56:28.554: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-41-188.ec2.internal 26656a1d-3c88-4e55-9129-517f0b3842f2 24064 0 2023-01-17 17:18:36 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-41-188.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-41-188.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"csi-mock-csi-mock-volumes-1369":"csi-mock-csi-mock-volumes-1369","ebs.csi.aws.com":"i-0f6e04c9eb7568cc9"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:36 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:32 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.3.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:41:03 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{},"f:volumesInUse":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.3.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f6e04c9eb7568cc9,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.3.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:56:05 +0000 UTC,LastTransitionTime:2023-01-17 17:40:52 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.41.188,},NodeAddress{Type:ExternalIP,Address:54.235.36.200,},NodeAddress{Type:Hostname,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-41-188.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-235-36-200.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec237328886d9a8cdd5a8a6503a5a6b4,SystemUUID:ec237328-886d-9a8c-dd5a-8a6503a5a6b4,BootID:d9256086-d9ef-44a5-8016-86239767825b,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/jessie-dnsutils@sha256:11e6a66017ba4e4b938c1612b7a54a3befcefd354796c04e1dba76873a13518e k8s.gcr.io/e2e-test-images/jessie-dnsutils:1.5],SizeBytes:253371792,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:20f25f275d46aa728f7615a1ccc19c78b2ed89435bf943a44b339f70f45508e6 k8s.gcr.io/e2e-test-images/httpd:2.4.39-2],SizeBytes:126894770,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/httpd@sha256:1b9d1b2f36cb2dbee1960e82a9344aeb11bd4c4c03abf5e1853e0559c23855e3 k8s.gcr.io/e2e-test-images/httpd:2.4.38-2],SizeBytes:123781643,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a k8s.gcr.io/sig-storage/csi-resizer:v1.1.0],SizeBytes:49176472,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nonewprivs@sha256:8ac1264691820febacf3aea5d152cbde6d10685731ec14966a9401c6f47a68ac k8s.gcr.io/e2e-test-images/nonewprivs:1.3],SizeBytes:7107254,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[kubernetes.io/csi/csi-mock-csi-mock-volumes-1369^4],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:28.554: INFO: Logging kubelet events for node ip-172-20-41-188.ec2.internal Jan 17 17:56:28.587: INFO: Logging pods the kubelet thinks is on node ip-172-20-41-188.ec2.internal Jan 17 17:56:28.629: INFO: test-ss-0 started at 2023-01-17 17:46:54 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container webserver ready: false, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-8wms8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: indexed-job-0-tfjcw started at 2023-01-17 17:53:07 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.629: INFO: rs-bnfrh started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:28.629: INFO: failed-jobs-history-limit-27899635-c5wgb started at 2023-01-17 17:55:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.629: INFO: affinity-clusterip-timeout-8rd8t started at 2023-01-17 17:56:23 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container affinity-clusterip-timeout ready: false, restart count 0 Jan 17 17:56:28.629: INFO: pvc-volume-tester-bg68c started at 2023-01-17 17:52:01 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container volume-tester ready: false, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-bfhwg started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: rs-jqsw7 started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:28.629: INFO: pod-submit-remove-a284d838-09fe-4289-9c01-b30c15621bef started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container nginx ready: false, restart count 0 Jan 17 17:56:28.629: INFO: cilium-6g78v started at 2023-01-17 17:18:37 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:28.629: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-kpz6g started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-rcrmc started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: pod-subpath-test-inlinevolume-6dzs started at 2023-01-17 17:51:29 +0000 UTC (2+2 container statuses recorded) Jan 17 17:56:28.629: INFO: Init container init-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:28.629: INFO: Init container test-init-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:28.629: INFO: Container test-container-subpath-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:28.629: INFO: Container test-container-volume-inlinevolume-6dzs ready: false, restart count 0 Jan 17 17:56:28.629: INFO: csi-mockplugin-attacher-0 started at 2023-01-17 17:51:42 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container csi-attacher ready: false, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-cldh2 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-6fx7k started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: failed-jobs-history-limit-27899634-tgtr5 started at 2023-01-17 17:54:00 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container c ready: false, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qx2n9 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-p4fb7 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: pod-subpath-test-inlinevolume-rnc8 started at 2023-01-17 17:51:31 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Init container init-volume-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:28.629: INFO: Container test-container-subpath-inlinevolume-rnc8 ready: false, restart count 0 Jan 17 17:56:28.629: INFO: security-context-79bf1a75-efe1-4735-b759-4b38a866b933 started at 2023-01-17 17:51:59 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container test-container ready: false, restart count 0 Jan 17 17:56:28.629: INFO: ebs-csi-node-pgms2 started at 2023-01-17 17:18:37 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:28.629: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:28.629: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:28.629: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-72zcp started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tnnhc started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.629: INFO: rs-jsbdw started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.629: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:28.629: INFO: csi-mockplugin-0 started at 2023-01-17 17:51:42 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:28.629: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:28.629: INFO: Container driver-registrar ready: true, restart count 0 Jan 17 17:56:28.629: INFO: Container mock ready: true, restart count 0 Jan 17 17:56:28.821: INFO: Latency metrics for node ip-172-20-41-188.ec2.internal Jan 17 17:56:28.821: INFO: Logging node info for node ip-172-20-48-3.ec2.internal Jan 17 17:56:28.874: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-48-3.ec2.internal 0d1be06e-7bec-46d4-9764-67f4f74de62f 24067 0 2023-01-17 17:18:34 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:t3.medium beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-48-3.ec2.internal kubernetes.io/os:linux kubernetes.io/role:node node-role.kubernetes.io/node: node.kubernetes.io/instance-type:t3.medium topology.ebs.csi.aws.com/zone:us-east-1a topology.hostpath.csi/node:ip-172-20-48-3.ec2.internal topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0f9c62f75c0462696"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kops-controller Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/node":{}}}} } {kubelet Update v1 2023-01-17 17:18:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {kube-controller-manager Update v1 2023-01-17 17:40:37 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.2.0/24\"":{}}}} } {kubelet Update v1 2023-01-17 17:40:49 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{},"f:topology.hostpath.csi/node":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:capacity":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.2.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0f9c62f75c0462696,Unschedulable:false,Taints:[]Taint{},ConfigSource:nil,PodCIDRs:[100.96.2.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3850088448 0} {<nil>} 3759852Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3745230848 0} {<nil>} 3657452Ki BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:51:24 +0000 UTC,LastTransitionTime:2023-01-17 17:40:49 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.48.3,},NodeAddress{Type:ExternalIP,Address:54.85.49.252,},NodeAddress{Type:Hostname,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-48-3.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-54-85-49-252.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec202aecc78fbe44c02ec2a9c59b8196,SystemUUID:ec202aec-c78f-be44-c02e-c2a9c59b8196,BootID:87a9d1d4-143e-430d-9436-6f0291e8261d,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[nginx@sha256:b8f2383a95879e1ae064940d9a200f67a6c79e710ed82ac42263397367e7cc4e nginx:latest],SizeBytes:141812353,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e k8s.gcr.io/e2e-test-images/agnhost:2.39],SizeBytes:126872991,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:c8e03f60afa90a28e4bb6ec9a8d0fc36d89de4b7475cf2d613afa793ec969fe0 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0],SizeBytes:56382329,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-provisioner@sha256:4e74c0492bceddc598de1c90cc5bc14dcda94cb49fa9c5bad9d117c4834b5e08 k8s.gcr.io/sig-storage/csi-provisioner:v2.2.1],SizeBytes:56379269,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-resizer@sha256:36c31f7e1f433c9634d24f876353e8646246d81a03c4e351202c2644daff1620 k8s.gcr.io/sig-storage/csi-resizer:v1.2.0],SizeBytes:53956308,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-snapshotter@sha256:ed98431376c9e944e19a465fe8ea944806714dd95416a0821096c78d66b579bd k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1],SizeBytes:53531990,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:c5be65d6679efabb969d9b019300d187437ae876f992c40911fd2892bbef3b36 k8s.gcr.io/sig-storage/csi-attacher:v3.2.0],SizeBytes:53500826,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-attacher@sha256:60ab9b3e6a030d3038c87c0d6bca2930f58d1d72823e6a4af09767dc83b696a2 k8s.gcr.io/sig-storage/csi-attacher:v3.2.1],SizeBytes:53499570,},ContainerImage{Names:[registry.k8s.io/coredns/coredns@sha256:017727efcfeb7d053af68e51436ce8e65edbc6ca573720afb4f79c8594036955 registry.k8s.io/coredns/coredns:v1.10.0],SizeBytes:50249337,},ContainerImage{Names:[k8s.gcr.io/sig-storage/hostpathplugin@sha256:232fe80174d60d520d36043103853a1d7ab4b7f3782cf43e45034f04ccda58ce k8s.gcr.io/sig-storage/hostpathplugin:v1.7.1],SizeBytes:29977921,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:a61d309da54641db41fb8f35718f744e9f730d4d0384f8c4b186ddc9f06cbd5f k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0],SizeBytes:19662887,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[k8s.gcr.io/sig-storage/csi-node-driver-registrar@sha256:2dee3fe5fe861bb66c3a4ac51114f3447a4cd35870e0f2e2b558c7a400d89589 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0],SizeBytes:18699436,},ContainerImage{Names:[k8s.gcr.io/sig-storage/mock-driver@sha256:a7b517f9e0f42ffade802eef9cefa271372386b85f55b702b493241e58459793 k8s.gcr.io/sig-storage/mock-driver:v4.1.0],SizeBytes:17680993,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[k8s.gcr.io/sig-storage/livenessprobe@sha256:529be2c9770add0cdd0c989115222ea9fc1be430c11095eb9f6dafcf98a36e2b k8s.gcr.io/sig-storage/livenessprobe:v2.4.0],SizeBytes:17175488,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/nginx@sha256:13616070e3f29de4417eee434a8ef472221c9e51b3d037b5a6b46cef08eb7443 k8s.gcr.io/e2e-test-images/nginx:1.14-2],SizeBytes:16032814,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:c318242786b139d18676b1c09a0ad7f15fc17f8f16a5b2e625cd0dc8c9703daf k8s.gcr.io/e2e-test-images/busybox:1.29-2],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/e2e-test-images/busybox@sha256:39e1e963e5310e9c313bad51523be012ede7b35bb9316517d19089a010356592 k8s.gcr.io/e2e-test-images/busybox:1.29-1],SizeBytes:1154361,},ContainerImage{Names:[k8s.gcr.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db k8s.gcr.io/pause:3.6 registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:28.875: INFO: Logging kubelet events for node ip-172-20-48-3.ec2.internal Jan 17 17:56:28.917: INFO: Logging pods the kubelet thinks is on node ip-172-20-48-3.ec2.internal Jan 17 17:56:28.960: INFO: rs-wpnqq started at 2023-01-17 17:49:03 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.960: INFO: Container donothing ready: false, restart count 0 Jan 17 17:56:28.960: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-sppgb started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.960: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-km28p started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-gkvnr started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-ndff8 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: rs-ckrvx started at 2023-01-17 17:49:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container donothing ready: true, restart count 0 Jan 17 17:56:28.961: INFO: hostexec-ip-172-20-48-3.ec2.internal-rv92s started at 2023-01-17 17:52:38 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:28.961: INFO: pod-4543964c-3ff6-45a4-a9df-fe5b24ebd2f5 started at 2023-01-17 17:55:09 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container write-pod ready: false, restart count 0 Jan 17 17:56:28.961: INFO: affinity-clusterip-timeout-t6b5w started at 2023-01-17 17:56:23 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container affinity-clusterip-timeout ready: false, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-92l44 started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: ebs-csi-node-jt9zf started at 2023-01-17 17:18:35 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:28.961: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:28.961: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:28.961: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:28.961: INFO: coredns-867df8f45c-spdt9 started at 2023-01-17 17:19:44 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container coredns ready: true, restart count 0 Jan 17 17:56:28.961: INFO: pod-subpath-test-preprovisionedpv-tzxp started at 2023-01-17 17:52:51 +0000 UTC (1+2 container statuses recorded) Jan 17 17:56:28.961: INFO: Init container test-init-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:28.961: INFO: Container test-container-subpath-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:28.961: INFO: Container test-container-volume-preprovisionedpv-tzxp ready: false, restart count 0 Jan 17 17:56:28.961: INFO: hostexec-ip-172-20-48-3.ec2.internal-5rgz7 started at 2023-01-17 17:55:04 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container agnhost-container ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-qp6dg started at 2023-01-17 17:41:06 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c9tnj started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cilium-4zwpz started at 2023-01-17 17:18:35 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:28.961: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-c8b47 started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-82r7j started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:28.961: INFO: cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a-tdjpx started at 2023-01-17 17:41:05 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:28.961: INFO: Container cleanup40-2ee17eee-356d-4d16-aed4-1f5d3883815a ready: true, restart count 0 Jan 17 17:56:29.088: INFO: Latency metrics for node ip-172-20-48-3.ec2.internal Jan 17 17:56:29.089: INFO: Logging node info for node ip-172-20-61-43.ec2.internal Jan 17 17:56:29.119: INFO: Node Info: &Node{ObjectMeta:{ip-172-20-61-43.ec2.internal 33875020-638d-4034-bf59-b0a2a5348ee2 22753 0 2023-01-17 17:14:43 +0000 UTC <nil> <nil> map[beta.kubernetes.io/arch:amd64 beta.kubernetes.io/instance-type:c5.large beta.kubernetes.io/os:linux failure-domain.beta.kubernetes.io/region:us-east-1 failure-domain.beta.kubernetes.io/zone:us-east-1a kops.k8s.io/kops-controller-pki: kubernetes.io/arch:amd64 kubernetes.io/hostname:ip-172-20-61-43.ec2.internal kubernetes.io/os:linux kubernetes.io/role:master node-role.kubernetes.io/control-plane: node-role.kubernetes.io/master: node.kubernetes.io/exclude-from-external-load-balancers: node.kubernetes.io/instance-type:c5.large topology.ebs.csi.aws.com/zone:us-east-1a topology.kubernetes.io/region:us-east-1 topology.kubernetes.io/zone:us-east-1a] map[csi.volume.kubernetes.io/nodeid:{"ebs.csi.aws.com":"i-0429ef268b48c8bd8"} node.alpha.kubernetes.io/ttl:0 volumes.kubernetes.io/controller-managed-attach-detach:true] [] [] [{kubelet Update v1 2023-01-17 17:14:43 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/instance-type":{},"f:beta.kubernetes.io/os":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:providerID":{}}} } {protokube Update v1 2023-01-17 17:15:06 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kops.k8s.io/kops-controller-pki":{},"f:node-role.kubernetes.io/control-plane":{},"f:node.kubernetes.io/exclude-from-external-load-balancers":{}}}} } {kops-controller Update v1 2023-01-17 17:15:17 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{"f:kubernetes.io/role":{},"f:node-role.kubernetes.io/master":{}}}} } {kube-controller-manager Update v1 2023-01-17 17:15:34 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"100.96.0.0/24\"":{}},"f:taints":{}}} } {kubelet Update v1 2023-01-17 17:16:21 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.ebs.csi.aws.com/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}} status}]},Spec:NodeSpec{PodCIDR:100.96.0.0/24,DoNotUseExternalID:,ProviderID:aws:///us-east-1a/i-0429ef268b48c8bd8,Unschedulable:false,Taints:[]Taint{Taint{Key:node-role.kubernetes.io/master,Value:,Effect:NoSchedule,TimeAdded:<nil>,},},ConfigSource:nil,PodCIDRs:[100.96.0.0/24],},Status:NodeStatus{Capacity:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{51527004160 0} {<nil>} 50319340Ki BinarySI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3724263424 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Allocatable:ResourceList{cpu: {{2 0} {<nil>} 2 DecimalSI},ephemeral-storage: {{46374303668 0} {<nil>} 46374303668 DecimalSI},hugepages-1Gi: {{0 0} {<nil>} 0 DecimalSI},hugepages-2Mi: {{0 0} {<nil>} 0 DecimalSI},memory: {{3619405824 0} {<nil>} BinarySI},pods: {{110 0} {<nil>} 110 DecimalSI},},Phase:,Conditions:[]NodeCondition{NodeCondition{Type:MemoryPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientMemory,Message:kubelet has sufficient memory available,},NodeCondition{Type:DiskPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasNoDiskPressure,Message:kubelet has no disk pressure,},NodeCondition{Type:PIDPressure,Status:False,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:14:43 +0000 UTC,Reason:KubeletHasSufficientPID,Message:kubelet has sufficient PID available,},NodeCondition{Type:Ready,Status:True,LastHeartbeatTime:2023-01-17 17:52:29 +0000 UTC,LastTransitionTime:2023-01-17 17:15:34 +0000 UTC,Reason:KubeletReady,Message:kubelet is posting ready status,},},Addresses:[]NodeAddress{NodeAddress{Type:InternalIP,Address:172.20.61.43,},NodeAddress{Type:ExternalIP,Address:52.206.34.92,},NodeAddress{Type:Hostname,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:InternalDNS,Address:ip-172-20-61-43.ec2.internal,},NodeAddress{Type:ExternalDNS,Address:ec2-52-206-34-92.compute-1.amazonaws.com,},},DaemonEndpoints:NodeDaemonEndpoints{KubeletEndpoint:DaemonEndpoint{Port:10250,},},NodeInfo:NodeSystemInfo{MachineID:ec2d232e68f43abe498b49dd0eca5567,SystemUUID:ec2d232e-68f4-3abe-498b-49dd0eca5567,BootID:2eec32d7-f71d-4c8e-bf1c-a6cafc00d6ea,KernelVersion:4.18.0-425.3.1.el8.x86_64,OSImage:Red Hat Enterprise Linux 8.7 (Ootpa),ContainerRuntimeVersion:docker://20.10.17,KubeletVersion:v1.23.15,KubeProxyVersion:v1.23.15,OperatingSystem:linux,Architecture:amd64,},Images:[]ContainerImage{ContainerImage{Names:[registry.k8s.io/etcdadm/etcd-manager@sha256:66a453db625abb268f4b3bbefc5a34a171d81e6e8796cecca54cfd71775c77c4 registry.k8s.io/etcdadm/etcd-manager:v3.0.20221209],SizeBytes:662209237,},ContainerImage{Names:[quay.io/cilium/cilium@sha256:06ce2b0a0a472e73334a7504ee5c5d8b2e2d7b72ef728ad94e564740dd505be5 quay.io/cilium/cilium:v1.12.5],SizeBytes:456569528,},ContainerImage{Names:[k8s.gcr.io/kube-apiserver-amd64:v1.23.15 registry.k8s.io/kube-apiserver-amd64:v1.23.15],SizeBytes:135240444,},ContainerImage{Names:[k8s.gcr.io/kube-controller-manager-amd64:v1.23.15 registry.k8s.io/kube-controller-manager-amd64:v1.23.15],SizeBytes:125046097,},ContainerImage{Names:[k8s.gcr.io/kube-proxy-amd64:v1.23.15 registry.k8s.io/kube-proxy-amd64:v1.23.15],SizeBytes:112336036,},ContainerImage{Names:[quay.io/cilium/operator@sha256:a6d24a006a6b92967ac90786b49bc1ac26e5477cf028cd1186efcfc2466484db quay.io/cilium/operator:v1.12.5],SizeBytes:110016956,},ContainerImage{Names:[registry.k8s.io/kops/kops-controller:1.26.0-beta.2],SizeBytes:104575115,},ContainerImage{Names:[registry.k8s.io/kops/dns-controller:1.26.0-beta.2],SizeBytes:102637092,},ContainerImage{Names:[registry.k8s.io/provider-aws/aws-ebs-csi-driver@sha256:f0c5de192d832e7c1daa6580d4a62e8fa6fc8eabc0917ae4cb7ed4d15e95b59e registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1],SizeBytes:94471559,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-provisioner@sha256:122bfb8c1edabb3c0edd63f06523e6940d958d19b3957dc7b1d6f81e9f1f6119 registry.k8s.io/sig-storage/csi-provisioner:v3.1.0],SizeBytes:57736070,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-resizer@sha256:9ebbf9f023e7b41ccee3d52afe39a89e3ddacdbb69269d583abfc25847cfd9e4 registry.k8s.io/sig-storage/csi-resizer:v1.4.0],SizeBytes:55464778,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-attacher@sha256:8b9c313c05f54fb04f8d430896f5f5904b6cb157df261501b29adc04d2b2dc7b registry.k8s.io/sig-storage/csi-attacher:v3.4.0],SizeBytes:54848595,},ContainerImage{Names:[k8s.gcr.io/kube-scheduler-amd64:v1.23.15 registry.k8s.io/kube-scheduler-amd64:v1.23.15],SizeBytes:53521430,},ContainerImage{Names:[registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:0103eee7c35e3e0b5cd8cdca9850dc71c793cdeb6669d8be7a89440da2d06ae4 registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1],SizeBytes:19582112,},ContainerImage{Names:[registry.k8s.io/sig-storage/livenessprobe@sha256:406f59599991916d2942d8d02f076d957ed71b541ee19f09fc01723a6e6f5932 registry.k8s.io/sig-storage/livenessprobe:v2.6.0],SizeBytes:17611980,},ContainerImage{Names:[registry.k8s.io/kops/kube-apiserver-healthcheck:1.26.0-beta.2],SizeBytes:8786911,},ContainerImage{Names:[registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db registry.k8s.io/pause:3.6],SizeBytes:682696,},},VolumesInUse:[],VolumesAttached:[]AttachedVolume{},Config:nil,},} Jan 17 17:56:29.119: INFO: Logging kubelet events for node ip-172-20-61-43.ec2.internal Jan 17 17:56:29.152: INFO: Logging pods the kubelet thinks is on node ip-172-20-61-43.ec2.internal Jan 17 17:56:29.188: INFO: kube-controller-manager-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container kube-controller-manager ready: true, restart count 4 Jan 17 17:56:29.188: INFO: kube-scheduler-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container kube-scheduler ready: true, restart count 0 Jan 17 17:56:29.188: INFO: dns-controller-58d7bbb845-2dhdd started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container dns-controller ready: true, restart count 0 Jan 17 17:56:29.188: INFO: cilium-operator-69c4b6896-cqj4l started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container cilium-operator ready: true, restart count 0 Jan 17 17:56:29.188: INFO: ebs-csi-controller-7d85b867bd-7kkr8 started at 2023-01-17 17:15:13 +0000 UTC (0+5 container statuses recorded) Jan 17 17:56:29.188: INFO: Container csi-attacher ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container csi-provisioner ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container csi-resizer ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:29.188: INFO: etcd-manager-events-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:29.188: INFO: etcd-manager-main-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container etcd-manager ready: true, restart count 0 Jan 17 17:56:29.188: INFO: kube-apiserver-ip-172-20-61-43.ec2.internal started at 2023-01-17 17:12:40 +0000 UTC (0+2 container statuses recorded) Jan 17 17:56:29.188: INFO: Container healthcheck ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container kube-apiserver ready: true, restart count 3 Jan 17 17:56:29.188: INFO: ebs-csi-node-mstzl started at 2023-01-17 17:15:13 +0000 UTC (0+3 container statuses recorded) Jan 17 17:56:29.188: INFO: Container ebs-plugin ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container liveness-probe ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container node-driver-registrar ready: true, restart count 0 Jan 17 17:56:29.188: INFO: cilium-dgpfw started at 2023-01-17 17:15:13 +0000 UTC (1+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Init container clean-cilium-state ready: true, restart count 0 Jan 17 17:56:29.188: INFO: Container cilium-agent ready: true, restart count 0 Jan 17 17:56:29.188: INFO: kops-controller-zhf96 started at 2023-01-17 17:15:13 +0000 UTC (0+1 container statuses recorded) Jan 17 17:56:29.188: INFO: Container kops-controller ready: true, restart count 0 Jan 17 17:56:29.296: INFO: Latency metrics for node ip-172-20-61-43.ec2.internal Jan 17 17:56:29.296: INFO: Waiting up to 3m0s for all (but 0) nodes to be ready �[1mSTEP�[0m: Destroying namespace "cronjob-3813" for this suite.
Filter through log files
go run hack/e2e.go -v --test --test_args='--ginkgo.focus=Kubernetes\se2e\ssuite\s\[sig\-apps\]\sDeployment\sshould\snot\sdisrupt\sa\scloud\sload\-balancer\'s\sconnectivity\sduring\srollout$'
/workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:161 Jan 17 17:46:34.723: Could not reach HTTP service through a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80 after 10m0s /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/service/util.go:29
[BeforeEach] [sig-apps] Deployment /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/framework.go:185 �[1mSTEP�[0m: Creating a kubernetes client Jan 17 17:33:50.146: INFO: >>> kubeConfig: /root/.kube/config �[1mSTEP�[0m: Building a namespace api object, basename deployment �[1mSTEP�[0m: Waiting for a default service account to be provisioned in namespace �[1mSTEP�[0m: Waiting for kube-root-ca.crt to be provisioned in namespace [BeforeEach] [sig-apps] Deployment /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:89 [It] should not disrupt a cloud load-balancer's connectivity during rollout /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:161 Jan 17 17:33:50.398: INFO: Creating Deployment "test-rolling-update-with-lb" Jan 17 17:33:50.471: INFO: deployment status: v1.DeploymentStatus{ObservedGeneration:0, Replicas:0, UpdatedReplicas:0, ReadyReplicas:0, AvailableReplicas:0, UnavailableReplicas:0, Conditions:[]v1.DeploymentCondition{v1.DeploymentCondition{Type:"Progressing", Status:"True", LastUpdateTime:time.Date(2023, time.January, 17, 17, 33, 50, 0, time.Local), LastTransitionTime:time.Date(2023, time.January, 17, 17, 33, 50, 0, time.Local), Reason:"NewReplicaSetCreated", Message:"Created new replica set \"test-rolling-update-with-lb-79bf9f6988\""}}, CollisionCount:(*int32)(nil)} Jan 17 17:33:52.503: INFO: deployment status: v1.DeploymentStatus{ObservedGeneration:1, Replicas:3, UpdatedReplicas:3, ReadyReplicas:2, AvailableReplicas:2, UnavailableReplicas:1, Conditions:[]v1.DeploymentCondition{v1.DeploymentCondition{Type:"Available", Status:"False", LastUpdateTime:time.Date(2023, time.January, 17, 17, 33, 50, 0, time.Local), LastTransitionTime:time.Date(2023, time.January, 17, 17, 33, 50, 0, time.Local), Reason:"MinimumReplicasUnavailable", Message:"Deployment does not have minimum availability."}, v1.DeploymentCondition{Type:"Progressing", Status:"True", LastUpdateTime:time.Date(2023, time.January, 17, 17, 33, 52, 0, time.Local), LastTransitionTime:time.Date(2023, time.January, 17, 17, 33, 50, 0, time.Local), Reason:"ReplicaSetUpdated", Message:"ReplicaSet \"test-rolling-update-with-lb-79bf9f6988\" is progressing."}}, CollisionCount:(*int32)(nil)} Jan 17 17:33:54.503: INFO: Creating a service test-rolling-update-with-lb with type=LoadBalancer and externalTrafficPolicy=Local in namespace deployment-5244 �[1mSTEP�[0m: creating a service deployment-5244/test-rolling-update-with-lb with type=LoadBalancer �[1mSTEP�[0m: waiting for loadbalancer for service deployment-5244/test-rolling-update-with-lb Jan 17 17:33:54.574: INFO: Waiting up to 15m0s for service "test-rolling-update-with-lb" to have a LoadBalancer Jan 17 17:36:34.644: INFO: Hitting the replica set's pods through the service's load balancer Jan 17 17:36:34.644: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:34.678: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:36.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:38.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:38.722: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:40.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:42.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:42.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:44.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:46.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:46.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:48.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:48.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:50.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:50.829: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:52.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:54.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:56.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:36:58.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:36:58.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:00.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:00.702: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:02.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:02.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:04.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:06.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:06.700: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:08.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:08.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:10.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:12.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:14.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:14.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:16.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:16.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:18.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:18.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:20.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:20.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:22.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:22.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:24.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:26.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:26.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:28.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:28.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:30.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:32.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:32.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:34.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:36.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:36.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:38.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:38.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:40.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:42.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:42.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:44.700: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:46.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:46.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:48.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:48.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:50.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:50.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:52.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:54.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:56.704: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:37:58.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:37:58.720: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:00.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:02.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:02.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:04.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:04.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:06.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:06.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:08.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:10.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:12.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:14.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:16.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:18.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:20.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:22.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:22.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:24.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:24.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:26.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:26.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:28.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:28.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:30.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:32.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:32.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:34.724: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:36.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:38.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:40.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:42.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:42.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:44.840: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:46.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:46.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:48.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:48.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:50.806: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:52.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:54.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:56.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:38:58.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:38:58.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:00.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:02.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:02.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:04.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:04.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:06.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:08.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:08.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:10.799: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:12.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:14.701: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:16.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:16.725: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:18.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:18.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:20.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:22.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:22.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:24.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:26.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:26.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:28.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:28.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:30.701: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:32.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:32.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:34.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:36.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:38.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:40.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:42.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:42.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:44.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:46.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:48.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:48.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:50.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:52.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:54.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:56.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:56.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:39:58.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:39:58.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:00.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:00.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:02.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:02.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:04.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:06.718: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:08.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:10.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:10.719: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:12.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:12.718: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:14.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:16.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:16.732: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:18.724: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:20.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:22.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:22.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:24.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:26.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:26.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:28.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:28.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:30.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:32.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:32.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:34.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:34.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:36.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:38.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:38.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:40.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:42.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:42.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:44.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:46.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:48.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:48.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:50.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:52.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:54.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:56.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:56.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:40:58.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:40:58.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:00.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:02.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:02.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:04.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:06.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:08.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:10.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:12.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:14.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:16.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:18.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:20.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:20.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:22.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:22.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:24.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:26.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:26.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:28.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:28.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:30.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:30.716: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:32.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:32.705: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:34.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:36.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:38.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:40.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:42.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:42.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:44.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:46.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:48.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:48.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:50.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:52.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:54.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:56.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:41:58.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:41:58.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:00.722: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:02.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:02.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:04.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:06.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:08.700: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:10.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:12.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:14.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:14.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:16.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:18.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:18.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:20.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:20.702: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:22.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:22.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:24.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:26.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:26.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:28.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:28.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:30.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:30.705: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:32.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:32.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:34.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:36.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:38.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:40.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:40.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:42.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:42.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:44.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:46.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:48.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:48.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:50.720: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:52.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:54.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:54.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:56.704: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:42:58.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:42:58.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:00.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:00.705: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:02.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:02.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:04.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:06.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:08.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:10.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:10.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:12.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:12.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:14.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:16.720: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:18.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:20.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:20.700: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:22.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:22.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:24.691: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:26.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:26.721: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:28.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:28.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:30.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:30.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:32.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:32.722: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:34.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:36.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:38.724: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:40.719: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:42.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:42.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:44.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:44.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:46.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:48.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:48.705: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:50.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:52.724: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:54.715: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:56.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:56.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:43:58.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:43:58.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:00.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:02.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:02.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:04.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:06.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:08.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:08.720: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:10.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:12.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:12.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:14.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:14.725: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:16.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:16.859: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:18.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:18.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:20.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:22.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:22.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:24.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:26.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:26.719: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:28.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:28.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:30.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:30.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:32.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:32.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:34.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:36.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:36.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:38.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:38.693: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:40.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:42.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:42.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:44.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:44.725: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:46.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:48.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:48.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:50.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:50.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:52.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:52.712: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:54.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:56.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:56.724: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:44:58.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:44:58.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:00.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:02.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:02.719: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:04.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:04.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:06.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:06.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:08.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:10.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:12.702: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:14.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:16.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:18.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:20.726: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:22.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:22.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:24.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:24.725: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:26.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:26.716: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:28.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:28.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:30.692: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:32.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:32.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:34.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:34.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:36.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:36.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:38.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:38.717: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:40.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:40.695: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:42.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:42.708: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:44.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:44.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:46.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:46.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:48.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:48.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:50.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:50.713: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:52.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:52.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:54.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:54.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:56.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:56.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:45:58.681: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:45:58.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:00.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:00.720: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:02.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:02.699: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:04.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:04.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:06.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:06.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:08.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:08.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:10.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:10.698: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:12.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:12.709: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:14.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:14.707: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:16.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:16.714: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:18.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:18.694: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:20.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:20.711: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:22.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:22.722: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:24.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:24.761: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:26.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:26.710: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:28.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:28.700: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:30.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:30.697: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:32.679: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:32.696: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:34.680: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:34.706: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:34.706: INFO: Poking "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello" Jan 17 17:46:34.723: INFO: Poke("http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello"): Get "http://a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80/echo?msg=hello": dial tcp: lookup a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com on 10.63.240.10:53: no such host Jan 17 17:46:34.723: FAIL: Could not reach HTTP service through a489f47983c4f492b94a81ec006f1b11-621451498.us-east-1.elb.amazonaws.com:80 after 10m0s Full Stack Trace k8s.io/kubernetes/test/e2e/framework/service.TestReachableHTTP(...) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/framework/service/util.go:29 k8s.io/kubernetes/test/e2e/apps.testRollingUpdateDeploymentWithLocalTrafficLoadBalancer(0x7b06bd0) /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:1393 +0x94c k8s.io/kubernetes/test/e2e/apps.glob..func4.12() /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:166 +0xe9 k8s.io/kubernetes/test/e2e.RunE2ETests(0x24c66d7) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e.go:133 +0x697 k8s.io/kubernetes/test/e2e.TestE2E(0x0) _output/dockerized/go/src/k8s.io/kubernetes/test/e2e/e2e_test.go:136 +0x19 testing.tRunner(0xc0007a0b60, 0x735e880) /usr/local/go/src/testing/testing.go:1259 +0x102 created by testing.(*T).Run /usr/local/go/src/testing/testing.go:1306 +0x35a [AfterEach] [sig-apps] Deployment /workspace/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/test/e2e/apps/deployment.go:83 Jan 17 17:46:34.832: INFO: Deployment "test-rolling-update-with-lb": &Deployment{ObjectMeta:{test-rolling-update-with-lb deployment-5244 05c9dc2e-cf78-4d89-93c4-3e47d128b189 15823 1 2023-01-17 17:33:50 +0000 UTC <nil> <nil> map[name:test-rolling-update-with-lb] map[deployment.kubernetes.io/revision:1] [] [] [{e2e.test Update apps/v1 2023-01-17 17:33:50 +0000 UTC FieldsV1 {"f:metadata":{"f:labels":{".":{},"f:name":{}}},"f:spec":{"f:progressDeadlineSeconds":{},"f:replicas":{},"f:revisionHistoryLimit":{},"f:selector":{},"f:strategy":{"f:rollingUpdate":{".":{},"f:maxSurge":{},"f:maxUnavailable":{}},"f:type":{}},"f:template":{"f:metadata":{"f:labels":{".":{},"f:iteration":{},"f:name":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"agnhost\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}} } {kube-controller-manager Update apps/v1 2023-01-17 17:40:50 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:deployment.kubernetes.io/revision":{}}},"f:status":{"f:availableReplicas":{},"f:conditions":{".":{},"k:{\"type\":\"Available\"}":{".":{},"f:lastTransitionTime":{},"f:lastUpdateTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Progressing\"}":{".":{},"f:lastTransitionTime":{},"f:lastUpdateTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}},"f:observedGeneration":{},"f:readyReplicas":{},"f:replicas":{},"f:updatedReplicas":{}}} status}]},Spec:DeploymentSpec{Replicas:*3,Selector:&v1.LabelSelector{MatchLabels:map[string]string{name: test-rolling-update-with-lb,},MatchExpressions:[]LabelSelectorRequirement{},},Template:{{ 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb] map[] [] [] []} {[] [] [{agnhost k8s.gcr.io/e2e-test-images/agnhost:2.39 [] [netexec --http-port=80 --udp-port=80] [] [] [] {map[] map[]} [] [] nil nil nil nil /dev/termination-log File IfNotPresent SecurityContext{Capabilities:nil,Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,} false false false}] [] Always 0xc002dd8ab8 <nil> ClusterFirst map[] <nil> false false false <nil> &PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,} [] &Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:name,Operator:In,Values:[test-rolling-update-with-lb],},LabelSelectorRequirement{Key:iteration,Operator:In,Values:[0],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},} default-scheduler [] [] <nil> nil [] <nil> <nil> <nil> map[] [] <nil> nil}},Strategy:DeploymentStrategy{Type:RollingUpdate,RollingUpdate:&RollingUpdateDeployment{MaxUnavailable:0,MaxSurge:1,},},MinReadySeconds:0,RevisionHistoryLimit:*10,Paused:false,ProgressDeadlineSeconds:*600,},Status:DeploymentStatus{ObservedGeneration:1,Replicas:3,UpdatedReplicas:3,AvailableReplicas:3,UnavailableReplicas:0,Conditions:[]DeploymentCondition{DeploymentCondition{Type:Progressing,Status:True,Reason:NewReplicaSetAvailable,Message:ReplicaSet "test-rolling-update-with-lb-79bf9f6988" has successfully progressed.,LastUpdateTime:2023-01-17 17:33:52 +0000 UTC,LastTransitionTime:2023-01-17 17:33:50 +0000 UTC,},DeploymentCondition{Type:Available,Status:True,Reason:MinimumReplicasAvailable,Message:Deployment has minimum availability.,LastUpdateTime:2023-01-17 17:40:50 +0000 UTC,LastTransitionTime:2023-01-17 17:40:50 +0000 UTC,},},ReadyReplicas:3,CollisionCount:nil,},} Jan 17 17:46:34.864: INFO: New ReplicaSet "test-rolling-update-with-lb-79bf9f6988" of Deployment "test-rolling-update-with-lb": &ReplicaSet{ObjectMeta:{test-rolling-update-with-lb-79bf9f6988 deployment-5244 29c05a22-c814-430f-8ef4-ad08f66ec9c2 15822 1 2023-01-17 17:33:50 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb pod-template-hash:79bf9f6988] map[deployment.kubernetes.io/desired-replicas:3 deployment.kubernetes.io/max-replicas:4 deployment.kubernetes.io/revision:1] [{apps/v1 Deployment test-rolling-update-with-lb 05c9dc2e-cf78-4d89-93c4-3e47d128b189 0xc0037013c7 0xc0037013c8}] [] [{kube-controller-manager Update apps/v1 2023-01-17 17:33:50 +0000 UTC FieldsV1 {"f:metadata":{"f:annotations":{".":{},"f:deployment.kubernetes.io/desired-replicas":{},"f:deployment.kubernetes.io/max-replicas":{},"f:deployment.kubernetes.io/revision":{}},"f:labels":{".":{},"f:iteration":{},"f:name":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"05c9dc2e-cf78-4d89-93c4-3e47d128b189\"}":{}}},"f:spec":{"f:replicas":{},"f:selector":{},"f:template":{"f:metadata":{"f:labels":{".":{},"f:iteration":{},"f:name":{},"f:pod-template-hash":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"agnhost\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}} } {kube-controller-manager Update apps/v1 2023-01-17 17:40:44 +0000 UTC FieldsV1 {"f:status":{"f:availableReplicas":{},"f:fullyLabeledReplicas":{},"f:observedGeneration":{},"f:readyReplicas":{},"f:replicas":{}}} status}]},Spec:ReplicaSetSpec{Replicas:*3,Selector:&v1.LabelSelector{MatchLabels:map[string]string{name: test-rolling-update-with-lb,pod-template-hash: 79bf9f6988,},MatchExpressions:[]LabelSelectorRequirement{},},Template:{{ 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb pod-template-hash:79bf9f6988] map[] [] [] []} {[] [] [{agnhost k8s.gcr.io/e2e-test-images/agnhost:2.39 [] [netexec --http-port=80 --udp-port=80] [] [] [] {map[] map[]} [] [] nil nil nil nil /dev/termination-log File IfNotPresent SecurityContext{Capabilities:nil,Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,} false false false}] [] Always 0xc003701488 <nil> ClusterFirst map[] <nil> false false false <nil> &PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,} [] &Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:name,Operator:In,Values:[test-rolling-update-with-lb],},LabelSelectorRequirement{Key:iteration,Operator:In,Values:[0],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},} default-scheduler [] [] <nil> nil [] <nil> <nil> <nil> map[] [] <nil> nil}},MinReadySeconds:0,},Status:ReplicaSetStatus{Replicas:3,FullyLabeledReplicas:3,ObservedGeneration:1,ReadyReplicas:3,AvailableReplicas:3,Conditions:[]ReplicaSetCondition{},},} Jan 17 17:46:34.896: INFO: Pod "test-rolling-update-with-lb-79bf9f6988-b75f5" is available: &Pod{ObjectMeta:{test-rolling-update-with-lb-79bf9f6988-b75f5 test-rolling-update-with-lb-79bf9f6988- deployment-5244 fa5c8997-e830-41d9-80f0-fdf456d5617c 15761 0 2023-01-17 17:33:50 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb pod-template-hash:79bf9f6988] map[] [{apps/v1 ReplicaSet test-rolling-update-with-lb-79bf9f6988 29c05a22-c814-430f-8ef4-ad08f66ec9c2 0xc003338997 0xc003338998}] [] [{kube-controller-manager Update v1 2023-01-17 17:33:50 +0000 UTC FieldsV1 {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:iteration":{},"f:name":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"29c05a22-c814-430f-8ef4-ad08f66ec9c2\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"agnhost\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}} } {kubelet Update v1 2023-01-17 17:40:48 +0000 UTC FieldsV1 {"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"172.20.52.234\"}":{".":{},"f:ip":{}}},"f:startTime":{}}} status}]},Spec:PodSpec{Volumes:[]Volume{Volume{Name:kube-api-access-k66xx,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:&ProjectedVolumeSource{Sources:[]VolumeProjection{VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:nil,ServiceAccountToken:&ServiceAccountTokenProjection{Audience:,ExpirationSeconds:*3607,Path:token,},},VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:&ConfigMapProjection{LocalObjectReference:LocalObjectReference{Name:kube-root-ca.crt,},Items:[]KeyToPath{KeyToPath{Key:ca.crt,Path:ca.crt,Mode:nil,},},Optional:nil,},ServiceAccountToken:nil,},VolumeProjection{Secret:nil,DownwardAPI:&DownwardAPIProjection{Items:[]DownwardAPIVolumeFile{DownwardAPIVolumeFile{Path:namespace,FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,Mode:nil,},},},ConfigMap:nil,ServiceAccountToken:nil,},},DefaultMode:*420,},StorageOS:nil,CSI:nil,Ephemeral:nil,},},},Containers:[]Container{Container{Name:agnhost,Image:k8s.gcr.io/e2e-test-images/agnhost:2.39,Command:[],Args:[netexec --http-port=80 --udp-port=80],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-k66xx,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,},},RestartPolicy:Always,TerminationGracePeriodSeconds:*0,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:default,DeprecatedServiceAccount:default,NodeName:ip-172-20-41-188.ec2.internal,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,},ImagePullSecrets:[]LocalObjectReference{},Hostname:,Subdomain:,Affinity:&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:name,Operator:In,Values:[test-rolling-update-with-lb],},LabelSelectorRequirement{Key:iteration,Operator:In,Values:[0],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},},SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},},HostAliases:[]HostAlias{},PriorityClassName:,Priority:*0,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,OS:nil,},Status:PodStatus{Phase:Running,Conditions:[]PodCondition{PodCondition{Type:Initialized,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:50 +0000 UTC,Reason:,Message:,},PodCondition{Type:Ready,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:51 +0000 UTC,Reason:,Message:,},PodCondition{Type:ContainersReady,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:51 +0000 UTC,Reason:,Message:,},PodCondition{Type:PodScheduled,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:50 +0000 UTC,Reason:,Message:,},},Message:,Reason:,HostIP:172.20.41.188,PodIP:172.20.52.234,StartTime:2023-01-17 17:33:50 +0000 UTC,ContainerStatuses:[]ContainerStatus{ContainerStatus{Name:agnhost,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2023-01-17 17:33:51 +0000 UTC,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:true,RestartCount:0,Image:k8s.gcr.io/e2e-test-images/agnhost:2.39,ImageID:docker-pullable://k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e,ContainerID:docker://1a0ad3e4d5dfe1cab81f51a10c292a926b64512a6062ee56ffdb7b3e7110a85d,Started:*true,},},QOSClass:BestEffort,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{PodIP{IP:172.20.52.234,},},EphemeralContainerStatuses:[]ContainerStatus{},},} Jan 17 17:46:34.896: INFO: Pod "test-rolling-update-with-lb-79bf9f6988-lh28m" is available: &Pod{ObjectMeta:{test-rolling-update-with-lb-79bf9f6988-lh28m test-rolling-update-with-lb-79bf9f6988- deployment-5244 315cc442-079a-4e35-a46a-982358eed83e 15821 0 2023-01-17 17:33:50 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb pod-template-hash:79bf9f6988] map[] [{apps/v1 ReplicaSet test-rolling-update-with-lb-79bf9f6988 29c05a22-c814-430f-8ef4-ad08f66ec9c2 0xc0033396a7 0xc0033396a8}] [] [{kube-controller-manager Update v1 2023-01-17 17:33:50 +0000 UTC FieldsV1 {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:iteration":{},"f:name":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"29c05a22-c814-430f-8ef4-ad08f66ec9c2\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"agnhost\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}} } {kubelet Update v1 2023-01-17 17:40:50 +0000 UTC FieldsV1 {"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"172.20.44.177\"}":{".":{},"f:ip":{}}},"f:startTime":{}}} status}]},Spec:PodSpec{Volumes:[]Volume{Volume{Name:kube-api-access-s97cf,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:&ProjectedVolumeSource{Sources:[]VolumeProjection{VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:nil,ServiceAccountToken:&ServiceAccountTokenProjection{Audience:,ExpirationSeconds:*3607,Path:token,},},VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:&ConfigMapProjection{LocalObjectReference:LocalObjectReference{Name:kube-root-ca.crt,},Items:[]KeyToPath{KeyToPath{Key:ca.crt,Path:ca.crt,Mode:nil,},},Optional:nil,},ServiceAccountToken:nil,},VolumeProjection{Secret:nil,DownwardAPI:&DownwardAPIProjection{Items:[]DownwardAPIVolumeFile{DownwardAPIVolumeFile{Path:namespace,FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,Mode:nil,},},},ConfigMap:nil,ServiceAccountToken:nil,},},DefaultMode:*420,},StorageOS:nil,CSI:nil,Ephemeral:nil,},},},Containers:[]Container{Container{Name:agnhost,Image:k8s.gcr.io/e2e-test-images/agnhost:2.39,Command:[],Args:[netexec --http-port=80 --udp-port=80],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-s97cf,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,},},RestartPolicy:Always,TerminationGracePeriodSeconds:*0,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:default,DeprecatedServiceAccount:default,NodeName:ip-172-20-34-144.ec2.internal,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,},ImagePullSecrets:[]LocalObjectReference{},Hostname:,Subdomain:,Affinity:&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:name,Operator:In,Values:[test-rolling-update-with-lb],},LabelSelectorRequirement{Key:iteration,Operator:In,Values:[0],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},},SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},},HostAliases:[]HostAlias{},PriorityClassName:,Priority:*0,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,OS:nil,},Status:PodStatus{Phase:Running,Conditions:[]PodCondition{PodCondition{Type:Initialized,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:50 +0000 UTC,Reason:,Message:,},PodCondition{Type:Ready,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:52 +0000 UTC,Reason:,Message:,},PodCondition{Type:ContainersReady,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:52 +0000 UTC,Reason:,Message:,},PodCondition{Type:PodScheduled,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2023-01-17 17:33:50 +0000 UTC,Reason:,Message:,},},Message:,Reason:,HostIP:172.20.34.144,PodIP:172.20.44.177,StartTime:2023-01-17 17:33:50 +0000 UTC,ContainerStatuses:[]ContainerStatus{ContainerStatus{Name:agnhost,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2023-01-17 17:33:51 +0000 UTC,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:true,RestartCount:0,Image:k8s.gcr.io/e2e-test-images/agnhost:2.39,ImageID:docker-pullable://k8s.gcr.io/e2e-test-images/agnhost@sha256:7e8bdd271312fd25fc5ff5a8f04727be84044eb3d7d8d03611972a6752e2e11e,ContainerID:docker://f3530f3bba09999fa415833cb8797a750ea8b89344636dc0361a8329ff8bad6a,Started:*true,},},QOSClass:BestEffort,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{PodIP{IP:172.20.44.177,},},EphemeralContainerStatuses:[]ContainerStatus{},},} Jan 17 17:46:34.896: INFO: Pod "test-rolling-update-with-lb-79bf9f6988-vfvgt" is available: &Pod{ObjectMeta:{test-rolling-update-with-lb-79bf9f6988-vfvgt test-rolling-update-with-lb-79bf9f6988- deployment-5244 0896a40f-815a-4092-a27b-8a3127d0f5e5 15629 0 2023-01-17 17:33:50 +0000 UTC <nil> <nil> map[iteration:0 name:test-rolling-update-with-lb pod-template-hash:79bf9f6988] map[] [{apps/v1 ReplicaSet test-rolling-update-with-lb-79bf9f6988 29c05a22-c814-430f-8ef4-ad08f66ec9c2 0xc0033398c7 0xc0033398c8}] [] [{kube-controller-manager Update v1 2023-01-17 17:33:50 +0000 UTC FieldsV1 {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:iteration":{},"f:name":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"29c05a22-c814-430f-8ef4-ad08f66ec9c2\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"agnhost\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}} } {kubelet Update v1 2023-01-17 17:40:44 +0000 UTC FieldsV1 {"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"172.20.39.105\"}":{".":{},"f:ip":{}}},"f:startTime":{}}} status}]},Spec:PodSpec{Volumes:[]Volume{Volume{Name:kube-api-access-mvhqt,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:&ProjectedVolumeSource{Sources:[]VolumeProjection{VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:nil,ServiceAccountToken:&ServiceAccountTokenProjection{Audience:,ExpirationSeconds:*3607,Path:token,},},VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:&ConfigMapProjection{LocalObjectReference:LocalObjectReference{Name:kube-root-ca.crt,},Items:[]KeyToPath{KeyToPath{Key:ca.crt,Path:ca.crt,Mode:nil,},},Optional:nil,},ServiceAccountToken:nil,},VolumeProjection{Secret:nil,DownwardAPI:&DownwardAPIProjection{Items:[]DownwardAPIVolumeFile{DownwardAPIVolumeFile{Path:namespace,FieldRef:&ObjectFieldSelector{