root@ubuntu:~/nginx_ingress# kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.44.0/deploy/static/provider/cloud/deploy.yaml namespace/ingress-nginx created serviceaccount/ingress-nginx created configmap/ingress-nginx-controller created clusterrole.rbac.authorization.k8s.io/ingress-nginx created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created role.rbac.authorization.k8s.io/ingress-nginx created rolebinding.rbac.authorization.k8s.io/ingress-nginx created service/ingress-nginx-controller-admission created service/ingress-nginx-controller created deployment.apps/ingress-nginx-controller created validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created serviceaccount/ingress-nginx-admission created clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created role.rbac.authorization.k8s.io/ingress-nginx-admission created rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created job.batch/ingress-nginx-admission-create created job.batch/ingress-nginx-admission-patch created root@ubuntu:~/nginx_ingress#
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx NAMESPACE NAME READY STATUS RESTARTS AGE ingress-nginx ingress-nginx-admission-create-hgm4k 0/1 Completed 0 85s ingress-nginx ingress-nginx-admission-patch-lfkxb 0/1 Completed 0 85s ingress-nginx ingress-nginx-controller-74fd5565fb-zzg2r 0/1 ErrImagePull 0 85s
[root@centos7 ~]# docker pull registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller:v0.44.0.3-8e83e7dc6-aliyun Trying to pull repository registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller ... v0.44.0.3-8e83e7dc6-aliyun: Pulling from registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller 540db60ca938: Extracting [> ] 32.77 kB/2.812 MB c55168ef6a25: Download complete 540db60ca938: Pull complete c55168ef6a25: Pull complete 0bd41aa48561: Pull complete 69c93a59eb9f: Pull complete 381fbf8157bf: Pull complete 6d47f4d54812: Pull complete 04e0b7fc260e: Pull complete d93e59e8461d: Pull complete 6051af2ddc2c: Pull complete a599e8ec0184: Pull complete b244a608171b: Pull complete 416a4eb62bd7: Pull complete c630cf50972d: Pull complete c524f7201e51: Pull complete 686e7dfd9da9: Pull complete 08e9a2d72a46: Pull complete 78d223b306c6: Pull complete dc2bcc4ee216: Pull complete Digest: sha256:7238b6230b678b312113a891ad5f9f7bbedc7839a913eaaee0def8aa748c3313 Status: Downloaded newer image for registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller:v0.44.0.3-8e83e7dc6-aliyun [root@centos7 ~]# [root@centos7 ~]# [root@centos7 ~]# docker tag registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller:v0.44.0.3-8e83e7dc6-aliyun k8s.gcr.io/ingress-nginx/controller:v0.44.0 [root@centos7 ~]#
还是失败
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-hgm4k 0/1 Completed 0 16m 10.244.129.161 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-lfkxb 0/1 Completed 0 16m 10.244.129.162 centos7 <none> <none> ingress-nginx ingress-nginx-controller-74fd5565fb-rhgh6 0/1 ErrImagePull 0 67s 10.244.129.164 centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl describe pods ingress-nginx-controller-74fd5565fb-rhgh6 -n ingress-nginx Name: ingress-nginx-controller-74fd5565fb-rhgh6 Namespace: ingress-nginx Priority: 0 Node: centos7/10.10.16.251 Start Time: Fri, 30 Jul 2021 16:46:10 +0800 Labels: app.kubernetes.io/component=controller app.kubernetes.io/instance=ingress-nginx app.kubernetes.io/name=ingress-nginx pod-template-hash=74fd5565fb Annotations: cni.projectcalico.org/podIP: 10.244.129.164/32 cni.projectcalico.org/podIPs: 10.244.129.164/32 Status: Pending IP: 10.244.129.164 IPs: IP: 10.244.129.164 Controlled By: ReplicaSet/ingress-nginx-controller-74fd5565fb Containers: controller: Container ID: Image: k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a Image ID: Ports: 80/TCP, 443/TCP, 8443/TCP Host Ports: 0/TCP, 0/TCP, 0/TCP Args: /nginx-ingress-controller --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller --election-id=ingress-controller-leader --ingress-class=nginx --configmap=$(POD_NAMESPACE)/ingress-nginx-controller --validating-webhook=:8443 --validating-webhook-certificate=/usr/local/certificates/cert --validating-webhook-key=/usr/local/certificates/key State: Waiting Reason: ErrImagePull Ready: False Restart Count: 0 Requests: cpu: 100m memory: 90Mi Liveness: http-get http://:10254/healthz delay=10s timeout=1s period=10s #success=1 #failure=5 Readiness: http-get http://:10254/healthz delay=10s timeout=1s period=10s #success=1 #failure=3 Environment: POD_NAME: ingress-nginx-controller-74fd5565fb-rhgh6 (v1:metadata.name) POD_NAMESPACE: ingress-nginx (v1:metadata.namespace) LD_PRELOAD: /usr/local/lib/libmimalloc.so Mounts: /usr/local/certificates/ from webhook-cert (ro) /var/run/secrets/kubernetes.io/serviceaccount from ingress-nginx-token-bqgxw (ro) Conditions: Type Status Initialized True Ready False ContainersReady False PodScheduled True Volumes: webhook-cert: Type: Secret (a volume populated by a Secret) SecretName: ingress-nginx-admission Optional: false ingress-nginx-token-bqgxw: Type: Secret (a volume populated by a Secret) SecretName: ingress-nginx-token-bqgxw Optional: false QoS Class: Burstable Node-Selectors: kubernetes.io/os=linux Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled <unknown> default-scheduler Successfully assigned ingress-nginx/ingress-nginx-controller-74fd5565fb-rhgh6 to centos7 Normal Pulling 31s (x3 over 103s) kubelet, centos7 Pulling image "k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a" Warning Failed 15s (x3 over 88s) kubelet, centos7 Failed to pull image "k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a": rpc error: code = Unknown desc = Get https://k8s.gcr.io/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) Warning Failed 15s (x3 over 88s) kubelet, centos7 Error: ErrImagePull Normal BackOff 2s (x3 over 87s) kubelet, centos7 Back-off pulling image "k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a" Warning Failed 2s (x3 over 87s) kubelet, centos7 Error: ImagePullBackOff root@ubuntu:~/nginx_ingress#
"k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a":
containerStatuses: - image: k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a imageID: "" lastState: {} name: controller ready: false restartCount: 0 started: false state:
解决问题： 通过修改Pod的yaml信息，修改镜像，将@sha ......这些删除保存后，Pod修复正常了
下载
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.44.0/deploy/static/provider/cloud/deploy.yaml
kubectl delete -f deploy.yaml
更改deploy.yaml
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-g6h4x 0/1 Completed 0 21s 10.244.129.169 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-4bkxs 0/1 Completed 1 21s 10.244.129.168 centos7 <none> <none> ingress-nginx ingress-nginx-controller-7474b7dff5-fd9vh 0/1 CrashLoopBackOff 1 21s 10.244.129.170 centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl describe pods ingress-nginx-controller-7474b7dff5-fd9vh -n ingress-nginx Name: ingress-nginx-controller-7474b7dff5-fd9vh Namespace: ingress-nginx Priority: 0 Node: centos7/10.10.16.251 Start Time: Fri, 30 Jul 2021 17:06:01 +0800 Labels: app.kubernetes.io/component=controller app.kubernetes.io/instance=ingress-nginx app.kubernetes.io/name=ingress-nginx pod-template-hash=7474b7dff5 Annotations: cni.projectcalico.org/podIP: 10.244.129.170/32 cni.projectcalico.org/podIPs: 10.244.129.170/32 Status: Running IP: 10.244.129.170 IPs: IP: 10.244.129.170 Controlled By: ReplicaSet/ingress-nginx-controller-7474b7dff5 Containers: controller: Container ID: docker://cf65c3e4e9a1a90ef3f68e6f9fd7c1510c047826cd907f768770c3adce55981c Image: k8s.gcr.io/ingress-nginx/controller:v0.44.0 Image ID: docker-pullable://registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller@sha256:7238b6230b678b312113a891ad5f9f7bbedc7839a913eaaee0def8aa748c3313 Ports: 80/TCP, 443/TCP, 8443/TCP Host Ports: 0/TCP, 0/TCP, 0/TCP Args: /nginx-ingress-controller --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller --election-id=ingress-controller-leader --ingress-class=nginx --configmap=$(POD_NAMESPACE)/ingress-nginx-controller --validating-webhook=:8443 --validating-webhook-certificate=/usr/local/certificates/cert --validating-webhook-key=/usr/local/certificates/key State: Waiting Reason: CrashLoopBackOff Last State: Terminated Reason: Error Exit Code: 1 Started: Fri, 30 Jul 2021 17:07:06 +0800 Finished: Fri, 30 Jul 2021 17:07:06 +0800 Ready: False Restart Count: 3 Requests: cpu: 100m memory: 90Mi Liveness: http-get http://:10254/healthz delay=10s timeout=1s period=10s #success=1 #failure=5 Readiness: http-get http://:10254/healthz delay=10s timeout=1s period=10s #success=1 #failure=3 Environment: POD_NAME: ingress-nginx-controller-7474b7dff5-fd9vh (v1:metadata.name) POD_NAMESPACE: ingress-nginx (v1:metadata.namespace) LD_PRELOAD: /usr/local/lib/libmimalloc.so Mounts: /usr/local/certificates/ from webhook-cert (ro) /var/run/secrets/kubernetes.io/serviceaccount from ingress-nginx-token-s2n66 (ro) Conditions: Type Status Initialized True Ready False ContainersReady False PodScheduled True Volumes: webhook-cert: Type: Secret (a volume populated by a Secret) SecretName: ingress-nginx-admission Optional: false ingress-nginx-token-s2n66: Type: Secret (a volume populated by a Secret) SecretName: ingress-nginx-token-s2n66 Optional: false QoS Class: Burstable Node-Selectors: kubernetes.io/os=linux Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled <unknown> default-scheduler Successfully assigned ingress-nginx/ingress-nginx-controller-7474b7dff5-fd9vh to centos7 Warning FailedMount 69s (x4 over 73s) kubelet, centos7 MountVolume.SetUp failed for volume "webhook-cert" : secret "ingress-nginx-admission" not found Normal Pulled 10s (x4 over 61s) kubelet, centos7 Container image "k8s.gcr.io/ingress-nginx/controller:v0.44.0" already present on machine Normal Created 9s (x4 over 60s) kubelet, centos7 Created container controller Normal Started 8s (x4 over 60s) kubelet, centos7 Started container controller Warning BackOff 1s (x8 over 56s) kubelet, centos7 Back-off restarting failed container root@ubuntu:~/nginx_ingress#
root@ubuntu:~/nginx_ingress# kubectl logs ingress-nginx-controller-7474b7dff5-fd9vh -n ingress-nginx standard_init_linux.go:178: exec user process caused "exec format error" root@ubuntu:~/nginx_ingress#
root@ubuntu:~/nginx_ingress# tail -f /var/log/syslog
原来是镜像有问题，不是arm64版本
MountVolume.SetUp failed for volume "webhook-cert" : secret "ingress-nginx-admission" not found
拉取arm64版本的镜像
[root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Using default tag: latest Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag latest not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Using default tag: latest Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag latest not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm Using default tag: latest Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm Tag latest not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm [root@centos7 ~]# docker search nginx-ingress-controller-arm INDEX NAME DESCRIPTION STARS OFFICIAL AUTOMATED docker.io docker.io/toolboc/nginx-ingress-controller-arm The nginx ingress controller for kubernete... 1 docker.io docker.io/anjia0532/nginx-ingress-controller-arm 0 docker.io docker.io/cdkbot/nginx-ingress-controller-arm64 0 docker.io docker.io/cruse/nginx-ingress-controller-arm 0 docker.io docker.io/cruse/nginx-ingress-controller-arm64 0 docker.io docker.io/gcmirrors/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm64 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm64 0 docker.io docker.io/imdingtalk/nginx-ingress-controller-arm 0 docker.io docker.io/johnbelamaric/nginx-ingress-controller-arm 0 docker.io docker.io/kodbasen/nginx-ingress-controller-armhf 0 docker.io docker.io/kontenapharos/nginx-ingress-controller-arm64 0 docker.io docker.io/ljfranklin/nginx-ingress-controller-arm 0 docker.io docker.io/mirrorgooglecontainers/nginx-ingress-controller-arm 0 docker.io docker.io/raynix/nginx-ingress-controller-arm This is the armhf/armv7 arch of nginx-ingr... 0 docker.io docker.io/shannonmpoole/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm64 0 docker.io docker.io/starlingx4arm/nginx-ingress-controller-arm64 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm quay.io/kubernetes-ingress-controller/ngin... 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm64 quay.io/kubernetes-ingress-controller/ngin... 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm64 0 [root@centos7 ~]#
[root@centos7 ~]# docker search nginx-ingress-controller-arm INDEX NAME DESCRIPTION STARS OFFICIAL AUTOMATED docker.io docker.io/toolboc/nginx-ingress-controller-arm The nginx ingress controller for kubernete... 1 docker.io docker.io/anjia0532/nginx-ingress-controller-arm 0 docker.io docker.io/cdkbot/nginx-ingress-controller-arm64 0 docker.io docker.io/cruse/nginx-ingress-controller-arm 0 docker.io docker.io/cruse/nginx-ingress-controller-arm64 0 docker.io docker.io/gcmirrors/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm64 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm64 0 docker.io docker.io/imdingtalk/nginx-ingress-controller-arm 0 docker.io docker.io/johnbelamaric/nginx-ingress-controller-arm 0 docker.io docker.io/kodbasen/nginx-ingress-controller-armhf 0 docker.io docker.io/kontenapharos/nginx-ingress-controller-arm64 0 docker.io docker.io/ljfranklin/nginx-ingress-controller-arm 0 docker.io docker.io/mirrorgooglecontainers/nginx-ingress-controller-arm 0 docker.io docker.io/raynix/nginx-ingress-controller-arm This is the armhf/armv7 arch of nginx-ingr... 0 docker.io docker.io/shannonmpoole/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm64 0 docker.io docker.io/starlingx4arm/nginx-ingress-controller-arm64 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm quay.io/kubernetes-ingress-controller/ngin... 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm64 quay.io/kubernetes-ingress-controller/ngin... 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm64 0 [root@centos7 ~]# docker pull docker.io/tnosaj/nginx-ingress-controller-arm64 Using default tag: latest Trying to pull repository docker.io/tnosaj/nginx-ingress-controller-arm64 ... manifest for docker.io/tnosaj/nginx-ingress-controller-arm64:latest not found [root@centos7 ~]# docker search --no-trunc nginx-ingress-controller-arm INDEX NAME DESCRIPTION STARS OFFICIAL AUTOMATED docker.io docker.io/toolboc/nginx-ingress-controller-arm The nginx ingress controller for kubernetes targeted to arm 1 docker.io docker.io/anjia0532/nginx-ingress-controller-arm 0 docker.io docker.io/cdkbot/nginx-ingress-controller-arm64 0 docker.io docker.io/cruse/nginx-ingress-controller-arm 0 docker.io docker.io/cruse/nginx-ingress-controller-arm64 0 docker.io docker.io/gcmirrors/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm 0 docker.io docker.io/gcrxio/nginx-ingress-controller-arm64 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm 0 docker.io docker.io/haojianxun/nginx-ingress-controller-arm64 0 docker.io docker.io/imdingtalk/nginx-ingress-controller-arm 0 docker.io docker.io/johnbelamaric/nginx-ingress-controller-arm 0 docker.io docker.io/kodbasen/nginx-ingress-controller-armhf 0 docker.io docker.io/kontenapharos/nginx-ingress-controller-arm64 0 docker.io docker.io/ljfranklin/nginx-ingress-controller-arm 0 docker.io docker.io/mirrorgooglecontainers/nginx-ingress-controller-arm 0 docker.io docker.io/raynix/nginx-ingress-controller-arm This is the armhf/armv7 arch of nginx-ingress-controller 0 docker.io docker.io/shannonmpoole/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm 0 docker.io docker.io/sqeven/nginx-ingress-controller-arm64 0 docker.io docker.io/starlingx4arm/nginx-ingress-controller-arm64 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm:0.20.0 0 docker.io docker.io/tnosaj/nginx-ingress-controller-arm64 quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.20.0 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm 0 docker.io docker.io/yeaheo/nginx-ingress-controller-arm64 0 [root@centos7 ~]# docker pull
[root@centos7 ingress-nginx]# docker image pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.20.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... 0.20.0: Pulling from quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 858153c1cb3d: Pull complete 1eb375441dff: Pull complete 0a156a786087: Pull complete fd40ef7c2de4: Pull complete 18253ca3bc50: Pull complete 1e71023a8a44: Pull complete 995d3d3d4e01: Pull complete 869ff88b0161: Pull complete fca547d841c6: Pull complete ebba24cff756: Pull complete 2b16555cfee5: Pull complete a03f7af61e60: Pull complete 8e31c097b808: Pull complete af2c52dfff05: Pull complete Digest: sha256:4b2f87140b85bef9b2c43f7c2a34f47d89e088533d9a784ff3acec8a30d53b96 Status: Downloaded newer image for quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.20.0 [root@centos7 ingress-nginx]#
[root@centos7 ~]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE docker.io/nginx latest 04bd8b4e0d30 8 days ago 126 MB docker.io/nginx <none> 36741ec2ad2b 3 weeks ago 126 MB docker.io/nginx <none> d868a2ccd9b1 5 weeks ago 126 MB docker.io/busybox latest 90441bfaac70 7 weeks ago 1.4 MB k8s.gcr.io/ingress-nginx/controller v0.44.0 b9d4d4783672 8 weeks ago 496 MB registry.cn-hangzhou.aliyuncs.com/acs/aliyun-ingress-controller v0.44.0.3-8e83e7dc6-aliyun b9d4d4783672 8 weeks ago 496 MB docker.io/calico/node v3.19.1 dd6d43d932df 2 months ago 116 MB docker.io/calico/cni v3.19.1 af1a713b9155 2 months ago 137 MB docker.io/calico/kube-controllers v3.19.1 8e4340f23bd3 2 months ago 57.2 MB docker.io/coredns/coredns coredns-arm64 31084f9a8be6 6 months ago 40.1 MB k8s.gcr.io/coredns 1.6.7 31084f9a8be6 6 months ago 40.1 MB docker.io/calico/pod2daemon-flexvol v3.19.1 f490e49f9222 8 months ago 9.41 MB docker.io/jettech/kube-webhook-certgen v1.5.1 a883f7fc3561 8 months ago 42.5 MB docker.io/mirrorgcrio/kube-proxy-arm64 v1.18.2 0cb5b8487aa0 15 months ago 115 MB docker.io/mirrorgcrio/kube-apiserver-arm64 v1.18.2 a9fb96db8751 15 months ago 168 MB docker.io/mirrorgcrio/kube-controller-manager-arm64 v1.18.2 c29900c3c83a 15 months ago 158 MB docker.io/mirrorgcrio/kube-scheduler-arm64 v1.18.2 5f966b4d926b 15 months ago 94.5 MB docker.io/mirrorgcrio/kube-proxy-arm64 v1.18.1 f782b1121865 15 months ago 115 MB k8s.gcr.io/kube-proxy v1.18.1 f782b1121865 15 months ago 115 MB registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy v1.18.0 b97481cce674 16 months ago 115 MB registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver v1.18.0 d8e9466cd29d 16 months ago 168 MB registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager v1.18.0 97a810e2bebf 16 months ago 158 MB registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler v1.18.0 574c00df71a4 16 months ago 94.5 MB quay.io/coreos/flannel v0.12.0-arm64 7cf4a417daaa 16 months ago 53.6 MB docker.io/mirrorgcrio/pause-arm64 3.2 2a060e2e7101 17 months ago 484 kB k8s.gcr.io/pause 3.2 2a060e2e7101 17 months ago 484 kB registry.cn-hangzhou.aliyuncs.com/google_containers/pause 3.2 2a060e2e7101 17 months ago 484 kB registry.cn-hangzhou.aliyuncs.com/google_containers/coredns 1.6.7 6e17ba78cf3e 18 months ago 41.5 MB docker.io/mirrorgcrio/etcd-arm64 3.4.3-0 ab707b0a0ea3 21 months ago 363 MB k8s.gcr.io/etcd 3.4.3-0 ab707b0a0ea3 21 months ago 363 MB docker.io/calico/node v3.1.4 715f56d3ab67 2 years ago 241 MB quay.io/calico/node v3.1.4 715f56d3ab67 2 years ago 241 MB quay.io/calico/typha v3.1.4 c0af4ae008ff 2 years ago 56.9 MB docker.io/calico/typha v3.1.4 c0af4ae008ff 2 years ago 56.9 MB docker.io/calico/cni v3.1.4 e454b0fb4bfd 2 years ago 69.4 MB quay.io/calico/cni v3.1.4 e454b0fb4bfd 2 years ago 69.4 MB quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 0.20.0 ac830cae0b1d 2 years ago 482 MB docker.io/busybox 1.28 89a35e2ebb6b 3 years ago 1.28 MB docker.io/tutum/dnsutils latest 6cd78a6d3256 6 years ago 200 MB [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.20.0 k8s.gcr.io/ingress-nginx/controller:v0.20.0 Error response from daemon: no such id: quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.20.0 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.20.0 k8s.gcr.io/ingress-nginx/controller:v0.20.0 [root@centos7 ~]#
https://quay.io/repository/kubernetes-ingress-controller/nginx-ingress-controller-arm64?tab=tags
[root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag v0.32.0 not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag v0.32.0 not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... 0.32.0: Pulling from quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 8a0637ca1ac9: Pull complete 7a872f102622: Pull complete 66b5fee23086: Pull complete b3a47bbe88d6: Pull complete 053ea9b2a50d: Pull complete d85db9e55b5c: Pull complete 88ef5ba60bae: Pull complete f5faa96a1188: Pull complete 0538267598c1: Pull complete cbf61254f8be: Pull complete 6252865c27ee: Pull complete 93975463be27: Pull complete Digest: sha256:1c12b35ae5db7699b43ad2b90a9e863cda6ab9e4863772a00d4463153f7fe46e Status: Downloaded newer image for quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 k8s.gcr.io/ingress-nginx/controller:v0.32.0 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 k8s.gcr.io/ingress-nginx/controller:0.32.0 [root@centos7 ~]# [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag v0.32.0 not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 "docker tag" requires exactly 2 argument(s). See 'docker tag --help'. Usage: docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:v0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... Pulling repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 Tag v0.32.0 not found in repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 [root@centos7 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 Trying to pull repository quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 ... 0.32.0: Pulling from quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64 8a0637ca1ac9: Pull complete 7a872f102622: Pull complete 66b5fee23086: Pull complete b3a47bbe88d6: Pull complete 053ea9b2a50d: Pull complete d85db9e55b5c: Pull complete 88ef5ba60bae: Pull complete f5faa96a1188: Pull complete 0538267598c1: Pull complete cbf61254f8be: Pull complete 6252865c27ee: Pull complete 93975463be27: Pull complete Digest: sha256:1c12b35ae5db7699b43ad2b90a9e863cda6ab9e4863772a00d4463153f7fe46e Status: Downloaded newer image for quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 k8s.gcr.io/ingress-nginx/controller:v0.32.0 [root@centos7 ~]# docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller-arm64:0.32.0 k8s.gcr.io/ingress-nginx/controller:0.32.0 [root@centos7 ~]#
kube-webhook-certgen镜像
镜像名称jettech/kube-webhook-certgen没有加arm64
[root@centos7 ~]# docker pull jettech/kube-webhook-certgen:v1.2.0 Trying to pull repository docker.io/jettech/kube-webhook-certgen ... v1.2.0: Pulling from docker.io/jettech/kube-webhook-certgen 9ff2acc3204b: Pulling fs layer 69e2f037cdb3: Pulling fs layer a7c6910c6251: Pulling fs layer error pulling image configuration: Get https://production.cloudflare.docker.com/registry-v2/docker/registry/v2/blobs/sha256/00/00e66932d79ff36ef32d86525ffd0471b355d48942b76d31733127feb1081bb0/data?verify=1627647728-EHreB4Qxwqzeryx98L0W898taBA%3D: dial tcp 104.18.123.25:443: i/o timeout [root@centos7 ~]# docker pull registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen:v1.2.2 Trying to pull repository registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen ... v1.2.2: Pulling from registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen 24f0c933cbef: Pull complete 69e2f037cdb3: Pull complete 271c9029b3e4: Pull complete Digest: sha256:4ecb4e11ce3b77a6ca002eeb88d58652d0a199cc802a0aae2128c760300ed4de Status: Downloaded newer image for registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen:v1.2.2 [root@centos7 ~]#
更改deploy.0.32.0.yaml的kube-webhook-certgen镜像
create -f deploy.0.32.0.yaml namespace/ingress-nginx created serviceaccount/ingress-nginx created configmap/ingress-nginx-controller created clusterrole.rbac.authorization.k8s.io/ingress-nginx created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created role.rbac.authorization.k8s.io/ingress-nginx created rolebinding.rbac.authorization.k8s.io/ingress-nginx created service/ingress-nginx-controller-admission created service/ingress-nginx-controller created deployment.apps/ingress-nginx-controller created validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created job.batch/ingress-nginx-admission-create created job.batch/ingress-nginx-admission-patch created role.rbac.authorization.k8s.io/ingress-nginx-admission created rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created serviceaccount/ingress-nginx-admission created root@ubuntu:~/nginx_ingress#
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 5s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 ContainerCreating 0 6s <none> centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 ContainerCreating 0 6s <none> centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 16s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 ContainerCreating 0 10s <none> centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 ContainerCreating 0 10s <none> centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 20s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 Completed 0 17s 10.244.129.176 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 ContainerCreating 0 17s <none> centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 27s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 Completed 0 19s 10.244.129.176 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 ContainerCreating 0 19s <none> centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 29s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 Completed 0 21s 10.244.129.176 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 ContainerCreating 0 21s <none> centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 31s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 Completed 0 29s 10.244.129.176 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 Completed 0 29s 10.244.129.177 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 ContainerCreating 0 39s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-nxs5r 0/1 Completed 0 31s 10.244.129.176 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-hvc4s 0/1 Completed 0 31s 10.244.129.177 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-jl6r9 0/1 Running 0 41s 10.244.129.179 centos7 <none> <none> root@ubuntu:~/nginx_ingress#
执行tag操作
[root@centos7 ~]# docker tag registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen:v1.2.2 jettech/kube-webhook-certgen:v1.2.2 [root@centos7 ~]# docker images | grep v1.2.2 docker.io/jettech/kube-webhook-certgen v1.2.2 ffc8e5abdec0 13 months ago 47.5 MB jettech/kube-webhook-certgen v1.2.2 5693ebf5622a 13 months ago 49 MB registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen v1.2.2 5693ebf5622a 13 months ago 49 MB [root@centos7 ~]#
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-p8kbd 0/1 CrashLoopBackOff 2 38s 10.244.129.180 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-kr7nr 0/1 Error 2 38s 10.244.129.181 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-svvjm 0/1 ContainerCreating 0 48s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl describe pods ingress-nginx-admission-create-p8kbd -n ingress-nginx Name: ingress-nginx-admission-create-p8kbd Namespace: ingress-nginx Priority: 0 Node: centos7/10.10.16.251 Start Time: Fri, 30 Jul 2021 19:46:45 +0800 Labels: app.kubernetes.io/component=admission-webhook app.kubernetes.io/instance=ingress-nginx app.kubernetes.io/managed-by=Helm app.kubernetes.io/name=ingress-nginx app.kubernetes.io/version=0.32.0 controller-uid=29c17f94-c00f-4c33-b8e7-f40326397679 helm.sh/chart=ingress-nginx-2.0.3 job-name=ingress-nginx-admission-create Annotations: cni.projectcalico.org/podIP: 10.244.129.180/32 cni.projectcalico.org/podIPs: 10.244.129.180/32 Status: Running IP: 10.244.129.180 IPs: IP: 10.244.129.180 Controlled By: Job/ingress-nginx-admission-create Containers: create: Container ID: docker://9dc49bde865eaf284a4c478a455ef628bed2e7cde21f205e366c417d1ac9cb0c Image: jettech/kube-webhook-certgen:v1.2.2 Image ID: docker-pullable://registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen@sha256:4ecb4e11ce3b77a6ca002eeb88d58652d0a199cc802a0aae2128c760300ed4de Port: <none> Host Port: <none> Args: create --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.ingress-nginx.svc --namespace=ingress-nginx --secret-name=ingress-nginx-admission State: Terminated Reason: Error Exit Code: 1 Started: Fri, 30 Jul 2021 19:47:37 +0800 Finished: Fri, 30 Jul 2021 19:47:37 +0800 Last State: Terminated Reason: Error Exit Code: 1 Started: Fri, 30 Jul 2021 19:47:09 +0800 Finished: Fri, 30 Jul 2021 19:47:09 +0800 Ready: False Restart Count: 3 Environment: <none> Mounts: /var/run/secrets/kubernetes.io/serviceaccount from ingress-nginx-admission-token-b4jv6 (ro) Conditions: Type Status Initialized True Ready False ContainersReady False PodScheduled True Volumes: ingress-nginx-admission-token-b4jv6: Type: Secret (a volume populated by a Secret) SecretName: ingress-nginx-admission-token-b4jv6 Optional: false QoS Class: BestEffort Node-Selectors: <none> Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s node.kubernetes.io/unreachable:NoExecute for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled <unknown> default-scheduler Successfully assigned ingress-nginx/ingress-nginx-admission-create-p8kbd to centos7 Normal Pulled 13s (x4 over 60s) kubelet, centos7 Container image "jettech/kube-webhook-certgen:v1.2.2" already present on machine Normal Created 12s (x4 over 60s) kubelet, centos7 Created container create Normal Started 12s (x4 over 59s) kubelet, centos7 Started container create Warning BackOff 11s (x5 over 57s) kubelet, centos7 Back-off restarting failed container root@ubuntu:~/nginx_ingress# ls deploy.0.20.0.yaml deploy.0.32.0.yaml deploy.yaml ingress-nginx nginxinc
[root@centos7 ~]# docker images | grep kube-webhook-certgen docker.io/jettech/kube-webhook-certgen v1.5.1 a883f7fc3561 8 months ago 42.5 MB docker.io/jettech/kube-webhook-certgen v1.2.2 ffc8e5abdec0 13 months ago 47.5 MB jettech/kube-webhook-certgen v1.2.2 5693ebf5622a 13 months ago 49 MB registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen v1.2.2 5693ebf5622a 13 months ago 49 MB docker.io/jettech/kube-webhook-certgen v1.2.0 00e66932d79f 21 months ago 47.5 MB [root@centos7 ~]#
更改镜像
root@ubuntu:~/nginx_ingress# cat deploy.0.32.0.yaml | grep kube-webhook-certgen image: jettech/kube-webhook-certgen:v1.5.1 image: jettech/kube-webhook-certgen:v1.5.1 root@ubuntu:~/nginx_ingress
root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-hx2m2 0/1 Completed 0 21s 10.244.129.182 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-nsn8g 0/1 Completed 0 21s 10.244.129.183 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-fqt59 0/1 Running 0 31s 10.244.129.184 centos7 <none> <none> root@ubuntu:~/nginx_ingress#
删除刚才tag的镜像
[root@centos7 ~]# docker rmi jettech/kube-webhook-certgen:v1.2.2 Untagged: jettech/kube-webhook-certgen:v1.2.2 Untagged: registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen@sha256:4ecb4e11ce3b77a6ca002eeb88d58652d0a199cc802a0aae2128c760300ed4de
[root@centos7 ~]# docker images | grep kube-webhook-certgen docker.io/jettech/kube-webhook-certgen v1.5.1 a883f7fc3561 8 months ago 42.5 MB docker.io/jettech/kube-webhook-certgen v1.2.2 ffc8e5abdec0 13 months ago 47.5 MB registry.cn-beijing.aliyuncs.com/fcu3dx/kube-webhook-certgen v1.2.2 5693ebf5622a 13 months ago 49 MB docker.io/jettech/kube-webhook-certgen v1.2.0 00e66932d79f 21 months ago 47.5 MB [root@centos7 ~]#
采用image: jettech/kube-webhook-certgen:v1.2.2
oot@ubuntu:~/nginx_ingress# cat deploy.0.32.0.yaml.bak | grep kube-webhook-certgen image: jettech/kube-webhook-certgen:v1.2.2 image: jettech/kube-webhook-certgen:v1.2.2 root@ubuntu:~/nginx_ingress# kubectl create -f deploy.0.32.0.yaml.bak namespace/ingress-nginx created serviceaccount/ingress-nginx created configmap/ingress-nginx-controller created clusterrole.rbac.authorization.k8s.io/ingress-nginx created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created role.rbac.authorization.k8s.io/ingress-nginx created rolebinding.rbac.authorization.k8s.io/ingress-nginx created service/ingress-nginx-controller-admission created service/ingress-nginx-controller created deployment.apps/ingress-nginx-controller created validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created job.batch/ingress-nginx-admission-create created job.batch/ingress-nginx-admission-patch created role.rbac.authorization.k8s.io/ingress-nginx-admission created rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created serviceaccount/ingress-nginx-admission created root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-bwrb2 0/1 Completed 0 13s 10.244.129.185 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-448z4 0/1 Completed 0 13s 10.244.129.186 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-97mr5 0/1 ContainerCreating 0 23s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-bwrb2 0/1 Completed 0 18s 10.244.129.185 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-448z4 0/1 Completed 0 18s 10.244.129.186 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-97mr5 0/1 ContainerCreating 0 28s <none> centos7 <none> <none> root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx ingress-nginx-admission-create-bwrb2 0/1 Completed 0 44s 10.244.129.185 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-448z4 0/1 Completed 0 44s 10.244.129.186 centos7 <none> <none> ingress-nginx ingress-nginx-controller-866488c6d4-97mr5 0/1 Running 0 54s 10.244.129.187 centos7 <none> <none> root@ubuntu:~/nginx_ingress#
root@ubuntu:~/nginx_ingress# kubectl get ingress No resources found in default namespace. root@ubuntu:~/nginx_ingress# kubectl create -f ingress.yaml ingress.extensions/example-ingress created root@ubuntu:~/nginx_ingress# kubectl get ingress NAME CLASS HOSTS ADDRESS PORTS AGE example-ingress <none> * 80 2s root@ubuntu:~/nginx_ingress#
[root@centos7 nginx_ingress]# kubectl exec -it ingress-nginx-controller-687ff7d857-2g5w7 -n ingress-nginx -- bash bash-5.1$ ps -elf PID USER TIME COMMAND 1 www-data 0:00 /usr/bin/dumb-init -- /nginx-ingress-controller --publish-service=ingress-nginx/ingress-nginx-controller --election-id=ingress-controller-leader --ingress-class=nginx --configmap=ingress-nginx/ingress-nginx-controller --validating 8 www-data 4:24 /nginx-ingress-controller --publish-service=ingress-nginx/ingress-nginx-controller --election-id=ingress-controller-leader --ingress-class=nginx --configmap=ingress-nginx/ingress-nginx-controller --validating-webhook=:8443 --valid 44 www-data 0:00 nginx: master process /usr/local/nginx/sbin/nginx -c /etc/nginx/nginx.conf 54 www-data 0:16 nginx: worker process 55 www-data 0:11 nginx: worker process 56 www-data 0:09 nginx: worker process 57 www-data 0:07 nginx: worker process 58 www-data 0:05 nginx: worker process 59 www-data 0:11 nginx: worker process 60 www-data 0:10 nginx: worker process 61 www-data 0:11 nginx: worker process 62 www-data 0:07 nginx: worker process 63 www-data 0:10 nginx: worker process 64 www-data 0:07 nginx: worker process 73 www-data 0:07 nginx: worker process 97 www-data 0:11 nginx: worker process 131 www-data 0:12 nginx: worker process 197 www-data 0:07 nginx: worker process 251 www-data 0:12 nginx: worker process 292 www-data 0:07 nginx: worker process
docker hub 拉取镜像
https://hub.docker.com/r/willdockerhub/ingress-nginx-controller/tags?page=1&ordering=last_updated
docker pull willdockerhub/ingress-nginx-controller:v0.47.0@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2
root@ubuntu:/opt/gopath/src/github.com/go-delve# docker pull willdockerhub/ingress-nginx-controller:v0.47.0 v0.47.0: Pulling from willdockerhub/ingress-nginx-controller 595b0fe564bb: Pull complete 69e3f0c5ecb6: Pull complete e518a94620d1: Pull complete b3a6ab783699: Pull complete 383815d8015d: Pull complete 7c877f431a9c: Pull complete cb9eba39d670: Pull complete 0b212051daa3: Pull complete 65c194f2a9d9: Pull complete 18c98e7b4a83: Pull complete db7b27880af9: Pull complete 7230e4462b8f: Pull complete bb23625bd2da: Pull complete 849860e12a6c: Pull complete Digest: sha256:a1e4efc107be0bb78f32eaec37bef17d7a0c81bec8066cdf2572508d21351d0b Status: Downloaded newer image for willdockerhub/ingress-nginx-controller:v0.47.0 docker.io/willdockerhub/ingress-nginx-controller:v0.47.0 root@ubuntu:/opt/gopath/src/github.com/go-delve# docker pull willdockerhub/ingress-nginx-controller:v0.47.0@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2: Pulling from willdockerhub/ingress-nginx-controller Digest: sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 Status: Downloaded newer image for willdockerhub/ingress-nginx-controller@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 docker.io/willdockerhub/ingress-nginx-controller:v0.47.0@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 root@ubuntu:/opt/gopath/src/github.com/go-delve# docker images | grep ingress-nginx-controller willdockerhub/ingress-nginx-controller v0.47.0 41af56ebe63c 8 weeks ago 270MB root@ubuntu:/opt/gopath/src/github.com/go-delve# docker rmi willdockerhub/ingress-nginx-controller Error: No such image: willdockerhub/ingress-nginx-controller root@ubuntu:/opt/gopath/src/github.com/go-delve# docker rmi 41af56ebe63c Untagged: willdockerhub/ingress-nginx-controller:v0.47.0 Untagged: willdockerhub/ingress-nginx-controller@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 Untagged: willdockerhub/ingress-nginx-controller@sha256:a1e4efc107be0bb78f32eaec37bef17d7a0c81bec8066cdf2572508d21351d0b Deleted: sha256:41af56ebe63cecb375bcbedb83b731c31a80e39227e38ba8acfb2937788a9b03 Deleted: sha256:8ec0d698311a578c982c965c0b1bf09b07035cd9c7af7f7bc7c0bdb91e09f264 Deleted: sha256:38d72f294461b57047d715ff8a4648308dcb5f087e3a001ef6e6b7ce9663ed12 Deleted: sha256:0ce98567785578c84b441f998b268178d2c204ced04bccda60e2cdbb6cccae14 Deleted: sha256:a1510495e6b0c41531d4f9c22559f552a4c137b9cd392cc36ff1826919c1800e Deleted: sha256:38aa11261b9f9c459a2e64ade67a43ee9fe7e60ce6d3dcae7fb2b6349dcb51b9 Deleted: sha256:fae4c04f218c4f4af534f0a86469a789a8faf1c6f0bf35b5f0ba580c7b194dde Deleted: sha256:2cae0d3709c3c58269ec2f9f2ff1ee5ffd5402f21e9c52d6a0851f1e09e45f08 Deleted: sha256:6aa97b8b567012c3a407daefdd4201df841ca2f49626539b9db3841970b9dd43 Deleted: sha256:830966fc361d759632e4b5e770285b4105d018c118c3d8ef48874f46afa53485 Deleted: sha256:432e788bfefc89af8f1cbbe14db794e9ca6a800659646a2d2b99c38ef71df4d3 Deleted: sha256:d709bf42c4338de8d6840d10a50aa461de8633142086ad27b14c2ef7ede65ca7 Deleted: sha256:99a508bae41ffb41a515310ec4fc8de35d618c0edb487e15f177469434cf631e Deleted: sha256:71bb23cac6b51a332b486b9b6eb0bafae58b791bb1ae8a108350105b1316035b Deleted: sha256:c55d5dbdab4094da9ba390de49be10dd3b42e990670236a81a792fd2c933fceb root@ubuntu:/opt/gopath/src/github.com/go-delve# docker pull willdockerhub/ingress-nginx-controller:v0.47.0@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2: Pulling from willdockerhub/ingress-nginx-controller 595b0fe564bb: Pull complete 69e3f0c5ecb6: Pull complete e518a94620d1: Pull complete b3a6ab783699: Pull complete 383815d8015d: Pull complete 7c877f431a9c: Pull complete cb9eba39d670: Pull complete 0b212051daa3: Pull complete 65c194f2a9d9: Pull complete 18c98e7b4a83: Pull complete db7b27880af9: Pull complete 7230e4462b8f: Pull complete bb23625bd2da: Pull complete 849860e12a6c: Pull complete Digest: sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 Status: Downloaded newer image for willdockerhub/ingress-nginx-controller@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 docker.io/willdockerhub/ingress-nginx-controller:v0.47.0@sha256:87644c17781d55c4f32342a75789a0b106f8a56b93fd6bafed801f096a0c1ef2 root@ubuntu:/opt/gopath/src/github.com/go-delve#
[root@centos7 ~]# docker tag 41af56ebe63c docker.io/willdockerhub/ingress-nginx-controller:v0.47.0 [root@centos7 ~]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE projects.registry.vmware.com/antrea/antrea-ubuntu latest 1db87c6991cb 4 days ago 376 MB docker.io/nginx latest 04bd8b4e0d30 13 days ago 126 MB docker.io/nginx <none> 36741ec2ad2b 4 weeks ago 126 MB docker.io/nginx <none> d868a2ccd9b1 6 weeks ago 126 MB docker.io/busybox latest 90441bfaac70 8 weeks ago 1.4 MB docker.io/willdockerhub/ingress-nginx-controller v0.47.0 41af56ebe63c 8 weeks ago 270 MB
root@ubuntu:~/nginx_ingress/ingress-nginx# kubectl get pods --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES default busybox 1/1 Running 146 6d2h 10.244.129.145 centos7 <none> <none> default example-foo-54dc4db9fc-42jz6 1/1 Running 0 5d 10.244.129.160 centos7 <none> <none> default web2-7cdf5dffb-26xrn 1/1 Running 0 4h57m 10.244.41.1 cloud <none> <none> default web3-c9654466d-xwb5j 1/1 Running 0 4h56m 10.244.41.2 cloud <none> <none> ingress-nginx ingress-nginx-admission-create-9qnqf 0/1 Completed 0 31s 10.244.129.147 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-dh6z5 0/1 Completed 2 31s 10.244.129.146 centos7 <none> <none> ingress-nginx ingress-nginx-controller-687ff7d857-2g5w7 0/1 Running 0 31s 10.244.129.148 centos7 <none> <none> kube-system calico-kube-controllers-5978c5f6b5-srmxk 1/1 Running 0 6h3m 10.244.129.130 centos7 <none> <none>
镜像编译
root@ubuntu:~/nginx_ingress/ingress-nginx# git branch * controller-0.32.0 main root@ubuntu:~/nginx_ingress/ingress-nginx# make image Makefile:49: *** mandatory variable ARCH is empty, either set it when calling the command or make sure 'go env GOARCH' works. Stop. root@ubuntu:~/nginx_ingress/ingress-nginx# go env GOARCH Command 'go' not found, but can be installed with: snap install go # version 1.16.6, or apt install golang-go apt install gccgo-go See 'snap info go' for additional versions. root@ubuntu:~/nginx_ingress/ingress-nginx# source ~/kata/go_source root@ubuntu:~/nginx_ingress/ingress-nginx# make image removing old image gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 Error: No such image: gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 Building docker image (arm64)... Sending build context to Docker daemon 314.4kB Step 1/26 : ARG BASE_IMAGE Step 2/26 : FROM ${BASE_IMAGE}
https://hub.docker.com/r/nginx/nginx-ingress
root@ubuntu:~/nginx_ingress# docker pull nginx/nginx-ingress Using default tag: latest latest: Pulling from nginx/nginx-ingress 69692152171a: Pull complete 30afc0b18f67: Pull complete 596b1d696923: Pull complete febe5bd23e98: Pull complete 8283eee92e2f: Pull complete 351ad75a6cfa: Pull complete ff0027f23312: Pull complete 18361787d4a4: Pull complete e1aba8353cbb: Pull complete f9a4438768c4: Pull complete Digest: sha256:a57fc7835c14b7294180f5c11ae6b81f2745189ef2adb57a5188952bf8aaa17a Status: Downloaded newer image for nginx/nginx-ingress:latest docker.io/nginx/nginx-ingress:latest
更改makefie
#REGISTRY ?= registry.cn-hangzhou.aliyuncs.com REGISTRY ?= gcr.io/k8s-staging-ingress-nginx BASE_IMAGE ?= nginx/nginx-ingress:latest #BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20210530-g6aab4c291@sha256:a7356029dd0c26cc3466bf7a27daec0f4df73aa14ca6c8b871a767022a812c0b
root@ubuntu:~/nginx_ingress/ingress-nginx# make image removing old image gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 Error: No such image: gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 Building docker image (arm64)... Sending build context to Docker daemon 314.4kB Step 1/26 : ARG BASE_IMAGE Step 2/26 : FROM ${BASE_IMAGE} ---> 1bc8d3e90493 Step 3/26 : ARG TARGETARCH ---> Running in a38b7cecf962 Removing intermediate container a38b7cecf962 ---> 82907dcb5eac Step 4/26 : ARG VERSION ---> Running in 2bbc41cfd95f Removing intermediate container 2bbc41cfd95f ---> 4af5e785d794 Step 5/26 : ARG COMMIT_SHA ---> Running in 6a15b09201e6 Removing intermediate container 6a15b09201e6 ---> 4f81e2a56697 Step 6/26 : ARG BUILD_ID=UNSET ---> Running in 442547077e9a Removing intermediate container 442547077e9a ---> 6bef413d8a3f Step 7/26 : LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes" ---> Running in 98bbd50506c1 Removing intermediate container 98bbd50506c1 ---> 9d152817550d Step 8/26 : LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/" ---> Running in 54b5dd1b3fb6 Removing intermediate container 54b5dd1b3fb6 ---> 46fa72f169f4 Step 9/26 : LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx" ---> Running in 91a827f938c2 Removing intermediate container 91a827f938c2 ---> 3c52fdc8b5b6 Step 10/26 : LABEL org.opencontainers.image.vendor="The Kubernetes Authors" ---> Running in eafdaccd5cde Removing intermediate container eafdaccd5cde ---> 4d70257b0317 Step 11/26 : LABEL org.opencontainers.image.licenses="Apache-2.0" ---> Running in d4bc9b0f20ed Removing intermediate container d4bc9b0f20ed ---> 4ad76e98505b Step 12/26 : LABEL org.opencontainers.image.version="${VERSION}" ---> Running in 51c6aadb88d6 Removing intermediate container 51c6aadb88d6 ---> a681c2c42f94 Step 13/26 : LABEL org.opencontainers.image.revision="${COMMIT_SHA}" ---> Running in 35b4b14e528d Removing intermediate container 35b4b14e528d ---> 832a7f7a0a83 Step 14/26 : LABEL build_id="${BUILD_ID}" ---> Running in bbe95edce2c8 Removing intermediate container bbe95edce2c8 ---> b748812a89f1 Step 15/26 : WORKDIR /etc/nginx ---> Running in 6a07f2e5148d Removing intermediate container 6a07f2e5148d ---> b97272f9578c Step 16/26 : RUN apk update && apk upgrade && apk add --no-cache diffutils && rm -rf /var/cache/apk/* ---> Running in ed7c4c424c75 standard_init_linux.go:211: exec user process caused "exec format error" The command '/bin/sh -c apk update && apk upgrade && apk add --no-cache diffutils && rm -rf /var/cache/apk/*' returned a non-zero code: 1 Makefile:65: recipe for target 'image' failed make: *** [image] Error 1
BASE_IMAGE ?= nginx/nginx-ingress:1.12.0-alpine
Step 16/26 : RUN apk update && apk upgrade && apk add --no-cache diffutils && rm -rf /var/cache/apk/* ---> Running in 48ca4ce62380 ERROR: Unable to lock database: Permission denied ERROR: Failed to open apk database: Permission denied The command '/bin/sh -c apk update && apk upgrade && apk add --no-cache diffutils && rm -rf /var/cache/apk/*' returned a non-zero code: 99 Makefile:66: recipe for target 'image' failed make: *** [image] Error 99
root@ubuntu:~/nginx_ingress/ingress-nginx# grep 'apk update' -rn * images/httpbin/rootfs/Dockerfile:20:RUN apk update images/nginx/rootfs/Dockerfile:20:RUN apk update images/nginx/rootfs/Dockerfile:37:RUN apk update rootfs/Dockerfile:36:RUN apk update
给这些dockerfile各自添加
USER root
fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/main/aarch64/APKINDEX.tar.gz fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/community/aarch64/APKINDEX.tar.gz (1/1) Installing diffutils (3.7-r0) Executing busybox-1.32.1-r6.trigger OK: 24 MiB in 44 packages Removing intermediate container 80b08ce52b0c ---> f54eeb0504ed Step 18/27 : COPY --chown=www-data:www-data etc /etc unable to convert uid/gid chown string to host mapping: can't
rootfs/Dockerfile
make image
WORKDIR /etc/nginx USER root RUN apk update && apk upgrade && apk add --no-cache diffutils && rm -rf /var/cache/apk/* RUN adduser -G www-data -D www-data
root@ubuntu:~/nginx_ingress/ingress-nginx# docker run -it --name myalpine alpine / # addgroup www-data && adduser -g www-data www-data adduser: group 'www-data' in use / # addgroup www-data && adduser -g www-data www-data addgroup: group 'www-data' in use / # adduser -g www-data www-data adduser: group 'www-data' in use / # adduser -G www-data www-data Changing password for www-data New password: Bad password: too short Retype password: passwd: password for www-data changed by root / #
/ # adduser -h adduser: option requires an argument: h BusyBox v1.31.1 () multi-call binary. Usage: adduser [OPTIONS] USER [GROUP] Create new user, or add USER to GROUP -h DIR Home directory -g GECOS GECOS field -s SHELL Login shell -G GRP Group -S Create a system user -D Don't assign a password -H Don't create home directory -u UID User id -k SKEL Skeleton directory (/etc/skel) / # adduser www-data www-data / #
# -g is the GID
addgroup -g 1000 groupname
# -u is the UID
# -D permits to create an user without password
adduser -u 1000 -G groupname -h /home/username -D username
root@ubuntu:~/nginx_ingress/ingress-nginx# docker run -it --name myalpine alpine / # adduser www-data www-data addgroup: unknown user www-data / # adduser www-data Changing password for www-data New password: passwd: password for www-data is unchanged / # addgroup AAA / # adduser AAA AAA addgroup: unknown user AAA / # adduser AAA adduser: group 'AAA' in use / # adduser AAA AAA addgroup: unknown user AAA / # adduser AAA2 Changing password for AAA2 New password: passwd: password for AAA2 is unchanged / # adduser -G AAA -D AAA / #
Step 23/28 : RUN bash -xeu -c ' writeDirs=( /etc/ingress-controller /etc/ingress-controller/ssl /etc/ingress-controller/auth /var/log /var/log/nginx ); for dir in "${writeDirs[@]}"; do mkdir -p ${dir}; chown -R www-data.www-data ${dir}; done' ---> Running in 9b15fae5075d /bin/sh: bash: not found The command '/bin/sh -c bash -xeu -c ' writeDirs=( /etc/ingress-controller /etc/ingress-controller/ssl /etc/ingress-controller/auth /var/log /var/log/nginx ); for dir in "${writeDirs[@]}"; do mkdir -p ${dir}; chown -R www-data.www-data ${dir}; done'' returned a non-zero code: 127 Makefile:64: recipe for target 'image' failed make: *** [image] Error 127
root@ubuntu:~/nginx_ingress/ingress-nginx# docker run -it --name myalpine alpine / # /bin/sh -c bash echo 'hello' echo: line 1: bash: not found / # /bin/sh -c echo 'hello' / #
Step 23/28 : RUN -xeu -c ' writeDirs=( /etc/ingress-controller /etc/ingress-controller/ssl /etc/ingress-controller/auth /var/log /var/log/nginx ); for dir in "${writeDirs[@]}"; do mkdir -p ${dir}; chown -R www-data.www-data ${dir}; done' ---> Running in 99bf9fd95154 /bin/sh: illegal option - The command '/bin/sh -c -xeu -c ' writeDirs=( /etc/ingress-controller /etc/ingress-controller/ssl /etc/ingress-controller/auth /var/log /var/log/nginx ); for dir in "${writeDirs[@]}"; do mkdir -p ${dir}; chown -R www-data.www-data ${dir}; done'' returned a non-zero code: 2 Makefile:64: recipe for target 'image' failed make: *** [image] Error 2
Alpine docker image doesn't have bash installed by default. You will need to add following commands to get bash:
RUN apk update && apk add bash
If youre using Alpine 3.3+ then you can just do
RUN apk add --no-cache bash
to keep docker image size small.
Step 25/29 : RUN apk add --no-cache libcap && setcap cap_net_bind_service=+ep /nginx-ingress-controller && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller && setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && apk del libcap ---> Running in f273f3f2be1e fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/main/aarch64/APKINDEX.tar.gz fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/community/aarch64/APKINDEX.tar.gz OK: 26 MiB in 46 packages /nginx-ingress-controller: OK Failed to set capabilities on file `/usr/local/nginx/sbin/nginx' (No such file or directory) usage: setcap [-q] [-v] [-n <rootid>] (-r|-|<caps>) <filename> [ ... (-r|-|<capsN>) <filenameN> ] Note <filename> must be a regular (non-symlink) file. The command '/bin/sh -c apk add --no-cache libcap && setcap cap_net_bind_service=+ep /nginx-ingress-controller && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller && setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && apk del libcap' returned a non-zero code: 1 Makefile:64: recipe for target 'image' failed make: *** [image] Error 1
docker export $(docker create nginx/nginx-ingress:1.12.0-alpine) | tar -C rootfs -xvf -
ls rootfs/usr/local/nginx/sbin/nginx ls: cannot access 'rootfs/usr/local/nginx/sbin/nginx': No such file or directory
root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine# ls rootfs/usr/local/nginx/sbin/nginx ls: cannot access 'rootfs/usr/local/nginx/sbin/nginx': No such file or directory root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine# cd rootfs/ root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine/rootfs# find ./ -name nginx ./usr/sbin/nginx ./usr/lib/nginx ./usr/share/licenses/nginx ./usr/share/nginx ./etc/logrotate.d/nginx ./etc/nginx ./etc/init.d/nginx ./var/cache/nginx ./var/lib/nginx ./var/log/nginx
root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine/rootfs# ln -sf usr/sbin/nginx usr/local/nginx/sbin/nginx ln: failed to create symbolic link 'usr/local/nginx/sbin/nginx': No such file or directory root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine/rootfs# mkdir -p usr/local/nginx/sbin/ root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine/rootfs# ln -sf usr/sbin/nginx usr/local/nginx/sbin/nginx root@ubuntu:~/nginx_ingress/ingress-nginx/nginx-ingress-alpine/rootfs#
编译成功
Step 26/30 : RUN apk add --no-cache libcap && setcap cap_net_bind_service=+ep /nginx-ingress-controller && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller && setcap cap_net_bind_service=+ep /usr/sbin/nginx && setcap -v cap_net_bind_service=+ep /usr/sbin/nginx && apk del libcap ---> Running in dc745c2d16ea fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/main/aarch64/APKINDEX.tar.gz fetch https://dl-cdn.alpinelinux.org/alpine/v3.13/community/aarch64/APKINDEX.tar.gz OK: 26 MiB in 46 packages /nginx-ingress-controller: OK /usr/sbin/nginx: OK (1/1) Purging libcap (2.46-r0) Executing busybox-1.32.1-r6.trigger OK: 25 MiB in 45 packages Removing intermediate container dc745c2d16ea ---> 38a626a270a9 Step 27/30 : USER www-data ---> Running in 1060c2e66635 Removing intermediate container 1060c2e66635 ---> 0ba9368ea0b2 Step 28/30 : RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log ---> Running in 6de1b15b3b2b Removing intermediate container 6de1b15b3b2b ---> 00a64d88b6a6 Step 29/30 : ENTRYPOINT ["/usr/bin/dumb-init", "--"] ---> Running in 93a9ffd8b328 Removing intermediate container 93a9ffd8b328 ---> d2a90af8ad0d Step 30/30 : CMD ["/nginx-ingress-controller"] ---> Running in 97fdfcade9c3 Removing intermediate container 97fdfcade9c3 ---> 03febdf7b291 Successfully built 03febdf7b291 Successfully tagged gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 root@ubuntu:~/nginx_ingress/ingress-nginx# find ./ -name gcr.io/k8s-staging-ingress-nginx find: warning: Unix filenames usually don't contain slashes (though pathnames do). That means that '-name ‘gcr.io/k8s-staging-ingress-nginx’' will probably evaluate to false all the time on this system. You might find the '-wholename' test more useful, or perhaps '-samefile'. Alternatively, if you are using GNU grep, you could use 'find ... -print0 | grep -FzZ ‘gcr.io/k8s-staging-ingress-nginx’'. root@ubuntu:~/nginx_ingress/ingress-nginx# dcoekr images | grep v0.48.1 dcoekr: command not found root@ubuntu:~/nginx_ingress/ingress-nginx# docker images | grep v0.48.1 gcr.io/k8s-staging-ingress-nginx/controller v0.48.1 03febdf7b291 2 minutes ago 153MB root@ubuntu:~/nginx_ingress/ingress-nginx#
运行报错
[root@centos7 nginx_ingress]# kubectl logs ingress-nginx-controller-577fbcd469-j4hwh -n ingress-nginx container_linux.go:235: starting container process caused "exec: "/usr/bin/dumb-init": stat /usr/bin/dumb-init: no such file or directory" [root@centos7 nginx_ingress]#
安装
RUN apk add --no-cache dumb-init
重新make image, 但是还是报错
[root@centos7 nginx_ingress]# kubectl logs ingress-nginx-controller-577fbcd469-f7b6x -n ingress-nginx ------------------------------------------------------------------------------- NGINX Ingress controller Release: v0.48.1 Build: git-f222c752b Repository: https://github.com/kubernetes/ingress-nginx.git nginx version: nginx/1.21.0 ------------------------------------------------------------------------------- I0805 03:04:06.395143 8 flags.go:211] "Watching for Ingress" class="nginx" W0805 03:04:06.395304 8 flags.go:216] Ingresses with an empty class will also be processed by this Ingress controller W0805 03:04:06.397176 8 client_config.go:614] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0805 03:04:06.397615 8 main.go:241] "Creating API client" host="https://10.96.0.1:443" I0805 03:04:06.410434 8 main.go:285] "Running in Kubernetes cluster" major="1" minor="18" git="v1.18.0" state="clean" commit="9e991415386e4cf155a24b1da15becaa390438d8" platform="linux/arm64" F0805 03:04:06.850503 8 ssl.go:389] unexpected error storing fake SSL Cert: could not create PEM certificate file /etc/ingress-controller/ssl/default-fake-certificate.pem: open /etc/ingress-controller/ssl/default-fake-certificate.pem: permission denied goroutine 1 [running]: k8s.io/klog/v2.stacks(0x400000e001, 0x40004d0600, 0x103, 0x1e1) k8s.io/klog/v2@v2.4.0/klog.go:1026 +0xac k8s.io/klog/v2.(*loggingT).output(0x227f360, 0x4000000003, 0x0, 0x0, 0x4000302cb0, 0x21d444e, 0x6, 0x185, 0x0) k8s.io/klog/v2@v2.4.0/klog.go:975 +0x17c k8s.io/klog/v2.(*loggingT).printf(0x227f360, 0x4000000003, 0x0, 0x0, 0x0, 0x0, 0x137dbe2, 0x2a, 0x40001a9c90, 0x1, ...) k8s.io/klog/v2@v2.4.0/klog.go:750 +0x12c k8s.io/klog/v2.Fatalf(...) k8s.io/klog/v2@v2.4.0/klog.go:1502 k8s.io/ingress-nginx/internal/net/ssl.GetFakeSSLCert(0xffffda33f7da) k8s.io/ingress-nginx/internal/net/ssl/ssl.go:389 +0x258 main.main() k8s.io/ingress-nginx/cmd/nginx/main.go:104 +0x3b4 goroutine 6 [chan receive]: k8s.io/klog/v2.(*loggingT).flushDaemon(0x227f360) k8s.io/klog/v2@v2.4.0/klog.go:1169 +0x70 created by k8s.io/klog/v2.init.0 k8s.io/klog/v2@v2.4.0/klog.go:417 +0x124 goroutine 142 [IO wait]: internal/poll.runtime_pollWait(0xffff57af4fc0, 0x72, 0xffffffffffffffff) runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x4000021298, 0x72, 0x1500, 0x1531, 0xffffffffffffffff) internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0x4000021280, 0x400046c000, 0x1531, 0x1531, 0x0, 0x0, 0x0) internal/poll/fd_unix.go:169 +0x158 net.(*netFD).Read(0x4000021280, 0x400046c000, 0x1531, 0x1531, 0x800, 0x4000, 0x40002eb908) net/fd_unix.go:202 +0x44 net.(*conn).Read(0x400021a008, 0x400046c000, 0x1531, 0x1531, 0x0, 0x0, 0x0) net/net.go:184 +0x74 crypto/tls.(*atLeastReader).Read(0x40003f5fe0, 0x400046c000, 0x1531, 0x1531, 0x40002eb928, 0x1b4ac, 0x40002eb938) crypto/tls/conn.go:760 +0x58 bytes.(*Buffer).ReadFrom(0x400029a258, 0x15eeba0, 0x40003f5fe0, 0x2d29f4, 0x1166420, 0x12d8440) bytes/buffer.go:204 +0xa4 crypto/tls.(*Conn).readFromUntil(0x400029a000, 0x15f0e60, 0x400021a008, 0x5, 0x400021a008, 0x701) crypto/tls/conn.go:782 +0xd0 crypto/tls.(*Conn).readRecordOrCCS(0x400029a000, 0x0, 0x0, 0x152b8) crypto/tls/conn.go:589 +0xd8 crypto/tls.(*Conn).readRecord(...) crypto/tls/conn.go:557 crypto/tls.(*Conn).Read(0x400029a000, 0x400027d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) crypto/tls/conn.go:1233 +0x11c bufio.(*Reader).Read(0x400042f500, 0x4000274118, 0x9, 0x9, 0x40003a4100, 0x494a18, 0x40003a4101) bufio/bufio.go:226 +0x238 io.ReadAtLeast(0x15eea00, 0x400042f500, 0x4000274118, 0x9, 0x9, 0x9, 0x40001a8060, 0x0, 0x15eed60) io/io.go:310 +0x78 io.ReadFull(...) io/io.go:329 golang.org/x/net/http2.readFrameHeader(0x4000274118, 0x9, 0x9, 0x15eea00, 0x400042f500, 0x0, 0x4000000000, 0xcb6fc, 0x4000218648) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:237 +0x5c golang.org/x/net/http2.(*Framer).ReadFrame(0x40002740e0, 0x40000191a0, 0x0, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:492 +0x74 golang.org/x/net/http2.(*clientConnReadLoop).run(0x40002ebfa8, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1819 +0xb0 golang.org/x/net/http2.(*ClientConn).readLoop(0x400043e180) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1741 +0x60 created by golang.org/x/net/http2.(*Transport).newClientConn golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:705 +0x570 [root@centos7 nginx_ingress]#
查看镜像
docker export $(docker create gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1) | tar -C rootfs2 -xvf -
正常的镜像有
default-fake-certificate.pem
这个文件
[root@centos7 nginx_ingress]# ls rootfs/etc/ingress-controller/ssl/default-fake-certificate.pem rootfs/etc/ingress-controller/ssl/default-fake-certificate.pem [root@centos7 nginx_ingress]# ls rootfs2/etc/ingress-controller/ssl/default-fake-certificate.pem ls: cannot access rootfs2/etc/ingress-controller/ssl/default-fake-certificate.pem: No such file or directory [root@centos7 nginx_ingress]# ls rootfs2/etc/ingress-controller/ssl/ [root@centos7 nginx_ingress]#
[root@centos7 nginx_ingress]# tree rootfs/etc/ingress-controller/ rootfs/etc/ingress-controller/ ├── auth └── ssl └── default-fake-certificate.pem 2 directories, 1 file [root@centos7 nginx_ingress]# tree rootfs2/etc/ingress-controller/ rootfs2/etc/ingress-controller/ ├── auth └── ssl 2 directories, 0 files [root@centos7 nginx_ingress]#
[root@centos7 nginx_ingress]# cp rootfs/etc/ingress-controller/ssl/default-fake-certificate.pem rootfs2/etc/ingress-controller/ssl/ You have new mail in /var/spool/mail/root [root@centos7 nginx_ingress]# tree rootfs2/etc/ingress-controller/ rootfs2/etc/ingress-controller/ ├── auth └── ssl └── default-fake-certificate.pem 2 directories, 1 file
tar -cvf rootfs2 controller.v0.48.1.tar
[root@centos7 nginx_ingress]# docker rmi -f 7cbaf4b10e58 Untagged: gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 Deleted: sha256:7cbaf4b10e587451a6e5989e7a74c01f32a89b23e7e0549ca99f02a3ea278fed [root@centos7 nginx_ingress]# docker import controller.v0.48.1.tar gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 sha256:f8ab455a0d47aedb1b9bdac7a9acab9558c6bc9eb1a1ee447f597dcc0fd6a46a [root@centos7 nginx_ingress]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE gcr.io/k8s-staging-ingress-nginx/controller v0.48.1 f8ab455a0d47 5 seconds ago 107 MB
运行同样报错
[root@centos7 nginx_ingress]# kubectl logs ingress-nginx-controller-577fbcd469-bn7tz -n ingress-nginx container_linux.go:235: starting container process caused "exec: "/nginx-ingress-controller": stat /nginx-ingress-controller: no such file or directory"
[root@centos7 nginx_ingress]# chroot rootfs2 /usr/bin/env -i SHELL=/bin/sh bash
bash-5.1# ls bin docker-entrypoint.sh media nginx.ingress.tmpl opt sbin usr dbg etc mnt nginx.tmpl proc srv var dev home nginx-ingress nginx.transportserver.tmpl root sys wait-shutdown docker-entrypoint.d lib nginx-ingress-controller nginx.virtualserver.tmpl run tmp bash-5.1# ls -al total 81848 drwxr-xr-x 20 root root 4096 Aug 5 03:08 . drwxr-xr-x 20 root root 4096 Aug 5 03:08 .. -rwxr-xr-x 1 root root 0 Aug 5 03:08 .dockerenv drwxr-xr-x 2 root root 4096 Aug 5 03:01 bin -rwxr-xr-x 1 www-data www-data 6946816 Aug 4 12:54 dbg drwxr-xr-x 4 root root 43 Aug 5 03:08 dev drwxr-xr-x 2 root root 115 Jun 16 01:12 docker-entrypoint.d -rwxrwxr-x 1 root root 1202 Jun 16 01:12 docker-entrypoint.sh drwxr-xr-x 19 root root 4096 Aug 5 03:08 etc drwxr-xr-x 3 root root 22 Aug 5 03:01 home drwxr-xr-x 7 root root 249 Aug 5 03:01 lib drwxr-xr-x 5 root root 44 Apr 14 10:28 media drwxr-xr-x 2 root root 6 Apr 14 10:28 mnt -rwxr-xr-x 1 nginx root 35586048 Jun 30 10:40 nginx-ingress -rwxr-xr-x 1 www-data www-data 36110336 Aug 4 12:54 nginx-ingress-controller -rw-r--r-- 1 nginx root 7576 Jun 30 10:41 nginx.ingress.tmpl -rw-r--r-- 1 nginx root 6736 Jun 30 10:41 nginx.tmpl -rw-r--r-- 1 nginx root 1240 Jun 30 10:41 nginx.transportserver.tmpl -rw-r--r-- 1 nginx root 9725 Jun 30 10:41 nginx.virtualserver.tmpl drwxr-xr-x 2 root root 6 Apr 14 10:28 opt dr-xr-xr-x 2 root root 6 Apr 14 10:28 proc drwx------ 2 root root 27 Aug 5 03:45 root drwxr-xr-x 2 root root 6 Apr 14 10:28 run drwxr-xr-x 2 root root 4096 Aug 5 03:01 sbin drwxr-xr-x 2 root root 6 Apr 14 10:28 srv drwxr-xr-x 2 root root 6 Apr 14 10:28 sys drwxrwxrwt 2 root root 6 Jun 16 01:12 tmp drwxr-xr-x 7 root root 66 Apr 14 10:28 usr drwxr-xr-x 12 root root 137 Apr 14 10:28 var -rwxr-xr-x 1 www-data www-data 5111808 Aug 4 12:54 wait-shutdown bash-5.1#
[root@centos7 nginx_ingress]# docker run -it --entrypoint bash gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 /usr/bin/docker-current: Error response from daemon: oci runtime error: container_linux.go:235: starting container process caused "exec: "bash": executable file not found in $PATH". [root@centos7 nginx_ingress]# docker run -it --entrypoint sh gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 /usr/bin/docker-current: Error response from daemon: oci runtime error: container_linux.go:235: starting container process caused "exec: "sh": executable file not found in $PATH". [root@centos7 nginx_ingress]# ls rootfs2/bin/ arch bbconfig chmod dd dumpkmap false fsync gzip ipcalc linux32 ls mknod mountpoint nice pipe_progress reformime run-parts sh su true watch ash busybox chown df echo fatattr getopt hostname kbd_mode linux64 lzop mktemp mpstat pidof printenv rev sed sleep sync umount zcat base64 cat cp dmesg ed fdflush grep ionice kill ln makemime more mv ping ps rm setpriv stat tar uname bash chgrp date dnsdomainname egrep fgrep gunzip iostat link login mkdir mount netstat ping6 pwd rmdir setserial stty touch usleep [root@centos7 nginx_ingress]# docker run -it --entrypoint bash gcr.io/k8s-staging-ingress-nginx/controller:v0.48.1 /usr/bin/docker-current: Error response from daemon: oci runtime error: container_linux.go:235: starting container process caused "exec: "bash": executable file not found in $PATH". [root@centos7 nginx_ingress]#
Please make sure to update it runAsUser: 101 or the ingress controller will not start (CrashLoopBackOff).
addgroup -Sg 101 www-data adduser -S -D -H -u 101 -h /usr/local/nginx -s /sbin/nologin -G www-data -g www-data www-data
root@ubuntu:~/nginx_ingress/ingress-nginx# docker run -it --name myalpine alpine / # adduser -u 101 -G www-data -D www-data adduser: unknown group www-data / # addgroup -Sg 101 www-data / # adduser -S -D -H -u 101 -G www-data -g www-data www-data / # adduser BusyBox v1.31.1 () multi-call binary. Usage: adduser [OPTIONS] USER [GROUP] Create new user, or add USER to GROUP -h DIR Home directory -g GECOS GECOS field -s SHELL Login shell -G GRP Group -S Create a system user -D Don't assign a password -H Don't create home directory -u UID User id -k SKEL Skeleton directory (/etc/skel) / #
Step 19/30 : RUN adduser -S -D -H -u 101 -G www-data -g www-data www-data ---> Running in c01a6cea0647 adduser: uid '101' in use The command '/bin/sh -c adduser -S -D -H -u 101 -G www-data -g www-data www-data' returned a non-zero code: 1 Makefile:64: recipe for target 'image' failed make: *** [image] Error 1
unexpected error storing fake SSL Cert: could not create PEM certificate file /etc/ingress-controller/ssl/default-fake-certificate.pem: open /etc/ingress-controller/ssl/default-fake-certificate.pem: permission denied
解决方法：
原因:随着版本提高，安全限制越来越高，对于权限的管理也越来越精细
node节点分别 chmod -R 777 /var/lib/docker 授权任意用户有docker临时文件的任意权限
又出现了新问题
[root@centos7 nginx_ingress]# kubectl logs ingress-nginx-controller-577fbcd469-52ssk -n ingress-nginx ------------------------------------------------------------------------------- NGINX Ingress controller Release: v0.48.1 Build: git-f222c752b Repository: https://github.com/kubernetes/ingress-nginx.git nginx version: nginx/1.21.0 ------------------------------------------------------------------------------- I0805 06:08:31.955876 8 flags.go:211] "Watching for Ingress" class="nginx" W0805 06:08:31.955989 8 flags.go:216] Ingresses with an empty class will also be processed by this Ingress controller W0805 06:08:31.957513 8 client_config.go:614] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0805 06:08:31.957853 8 main.go:241] "Creating API client" host="https://10.96.0.1:443" I0805 06:08:31.967598 8 main.go:285] "Running in Kubernetes cluster" major="1" minor="18" git="v1.18.0" state="clean" commit="9e991415386e4cf155a24b1da15becaa390438d8" platform="linux/arm64" I0805 06:08:32.085512 8 main.go:105] "SSL fake certificate created" file="/etc/ingress-controller/ssl/default-fake-certificate.pem" I0805 06:08:32.086619 8 main.go:115] "Enabling new Ingress features available since Kubernetes v1.18" W0805 06:08:32.089227 8 main.go:127] No IngressClass resource with name nginx found. Only annotation will be used. I0805 06:08:32.117168 8 ssl.go:532] "loading tls certificate" path="/usr/local/certificates/cert" key="/usr/local/certificates/key" F0805 06:08:32.151513 8 nginx.go:191] Error creating file watchers: lstat /etc/nginx/geoip/: no such file or directory goroutine 1 [running]: k8s.io/klog/v2.stacks(0x4000230001, 0x400061c400, 0x7d, 0x1e1) k8s.io/klog/v2@v2.4.0/klog.go:1026 +0xac k8s.io/klog/v2.(*loggingT).output(0x227f360, 0x4000000003, 0x0, 0x0, 0x4000575f80, 0x21cebf8, 0x8, 0xbf, 0x0) k8s.io/klog/v2@v2.4.0/klog.go:975 +0x17c k8s.io/klog/v2.(*loggingT).printf(0x227f360, 0x4000000003, 0x0, 0x0, 0x0, 0x0, 0x136dbd1, 0x20, 0x4000bcfc70, 0x1, ...) k8s.io/klog/v2@v2.4.0/klog.go:750 +0x12c k8s.io/klog/v2.Fatalf(...) k8s.io/klog/v2@v2.4.0/klog.go:1502 k8s.io/ingress-nginx/internal/ingress/controller.NewNGINXController(0x4000203680, 0x1650460, 0x4000802f00, 0x4000802f00) k8s.io/ingress-nginx/internal/ingress/controller/nginx.go:191 +0x9e0 main.main() k8s.io/ingress-nginx/cmd/nginx/main.go:167 +0x744 goroutine 18 [chan receive]: k8s.io/klog/v2.(*loggingT).flushDaemon(0x227f360) k8s.io/klog/v2@v2.4.0/klog.go:1169 +0x70 created by k8s.io/klog/v2.init.0 k8s.io/klog/v2@v2.4.0/klog.go:417 +0x124 goroutine 194 [IO wait]: internal/poll.runtime_pollWait(0xffff8e878fc0, 0x72, 0xffffffffffffffff) runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x400003ba98, 0x72, 0x2c00, 0x2c51, 0xffffffffffffffff) internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0x400003ba80, 0x40005be000, 0x2c51, 0x2c51, 0x0, 0x0, 0x0) internal/poll/fd_unix.go:169 +0x158 net.(*netFD).Read(0x400003ba80, 0x40005be000, 0x2c51, 0x2c51, 0x800, 0x4000, 0x4000213908) net/fd_unix.go:202 +0x44 net.(*conn).Read(0x4000558b38, 0x40005be000, 0x2c51, 0x2c51, 0x0, 0x0, 0x0) net/net.go:184 +0x74 crypto/tls.(*atLeastReader).Read(0x40007285c0, 0x40005be000, 0x2c51, 0x2c51, 0x4000213928, 0x1b4ac, 0x4000213938) crypto/tls/conn.go:760 +0x58 bytes.(*Buffer).ReadFrom(0x400028c5d8, 0x15eeba0, 0x40007285c0, 0x2d29f4, 0x1166420, 0x12d8440) bytes/buffer.go:204 +0xa4 crypto/tls.(*Conn).readFromUntil(0x400028c380, 0x15f0e60, 0x4000558b38, 0x5, 0x4000558b38, 0x301) crypto/tls/conn.go:782 +0xd0 crypto/tls.(*Conn).readRecordOrCCS(0x400028c380, 0x0, 0x0, 0x152b8) crypto/tls/conn.go:589 +0xd8 crypto/tls.(*Conn).readRecord(...) crypto/tls/conn.go:557 crypto/tls.(*Conn).Read(0x400028c380, 0x400064f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) crypto/tls/conn.go:1233 +0x11c bufio.(*Reader).Read(0x4000401020, 0x4000648118, 0x9, 0x9, 0x400025ab00, 0x494a18, 0x400025ab01) bufio/bufio.go:226 +0x238 io.ReadAtLeast(0x15eea00, 0x4000401020, 0x4000648118, 0x9, 0x9, 0x9, 0x4000216050, 0x0, 0x15eed60) io/io.go:310 +0x78 io.ReadFull(...) io/io.go:329 golang.org/x/net/http2.readFrameHeader(0x4000648118, 0x9, 0x9, 0x15eea00, 0x4000401020, 0x0, 0x4000000000, 0xcb6fc, 0x4000561048) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:237 +0x5c golang.org/x/net/http2.(*Framer).ReadFrame(0x40006480e0, 0x40004059e0, 0x0, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:492 +0x74 golang.org/x/net/http2.(*clientConnReadLoop).run(0x4000213fa8, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1819 +0xb0 golang.org/x/net/http2.(*ClientConn).readLoop(0x40003a2000) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1741 +0x60 created by golang.org/x/net/http2.(*Transport).newClientConn golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:705 +0x570 goroutine 209 [sleep]: time.Sleep(0x12a05f200) runtime/time.go:188 +0xc0 k8s.io/ingress-nginx/internal/ingress/metric.(*collector).Start.func1(0x4000802f00) k8s.io/ingress-nginx/internal/ingress/metric/main.go:140 +0x30 created by k8s.io/ingress-nginx/internal/ingress/metric.(*collector).Start k8s.io/ingress-nginx/internal/ingress/metric/main.go:139 +0x16c goroutine 210 [chan receive]: k8s.io/ingress-nginx/internal/ingress/metric/collectors.namedProcess.Start(0x4000c22280, 0x4000bd66c0, 0x4000c37200, 0x40005560e0, 0x4000556150, 0x40005561c0, 0x40005563f0, 0x4000556460, 0x40005564d0, 0x4000556540) k8s.io/ingress-nginx/internal/ingress/metric/collectors/process.go:185 +0x88 created by k8s.io/ingress-nginx/internal/ingress/metric.(*collector).Start k8s.io/ingress-nginx/internal/ingress/metric/main.go:143 +0x194 goroutine 211 [IO wait]: internal/poll.runtime_pollWait(0xffff8e878ee0, 0x72, 0x0) runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x400003bd18, 0x72, 0x0, 0x0, 0x134d382) internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0x400003bd00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) internal/poll/fd_unix.go:384 +0x184 net.(*netFD).accept(0x400003bd00, 0x0, 0x0, 0x0) net/fd_unix.go:238 +0x2c net.(*UnixListener).accept(0x4000544cc0, 0x0, 0x0, 0x0) net/unixsock_posix.go:162 +0x2c net.(*UnixListener).Accept(0x4000544cc0, 0x0, 0x0, 0x0, 0x0) net/unixsock.go:260 +0x5c k8s.io/ingress-nginx/internal/ingress/metric/collectors.(*SocketCollector).Start(0x4000574380) k8s.io/ingress-nginx/internal/ingress/metric/collectors/socket.go:328 +0x34 created by k8s.io/ingress-nginx/internal/ingress/metric.(*collector).Start k8s.io/ingress-nginx/internal/ingress/metric/main.go:144 +0x1b8 goroutine 212 [IO wait]: internal/poll.runtime_pollWait(0xffff8e878e00, 0x72, 0x0) runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x400070e118, 0x72, 0x0, 0x0, 0x134d382) internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0x400070e100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) internal/poll/fd_unix.go:384 +0x184 net.(*netFD).accept(0x400070e100, 0xffffb55607d0, 0x0, 0x128ec40) net/fd_unix.go:238 +0x2c net.(*TCPListener).accept(0x4000716080, 0x10fb14, 0x30, 0x124e7c0) net/tcpsock_posix.go:139 +0x2c net.(*TCPListener).Accept(0x4000716080, 0x4000476210, 0x40004c2000, 0x40004bedb8, 0x112a540) net/tcpsock.go:261 +0x5c net/http.(*Server).Serve(0x40004c2000, 0x162e240, 0x4000716080, 0x0, 0x0) net/http/server.go:2930 +0x218 net/http.(*Server).ListenAndServe(0x40004c2000, 0x40004c2000, 0x40004befb8) net/http/server.go:2859 +0xac main.registerProfiler() k8s.io/ingress-nginx/cmd/nginx/main.go:344 +0x350 created by main.main k8s.io/ingress-nginx/cmd/nginx/main.go:164 +0x8e4 goroutine 213 [chan receive]: k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0x4000802f40) k8s.io/apimachinery@v0.20.2/pkg/watch/mux.go:219 +0x4c created by k8s.io/apimachinery/pkg/watch.NewBroadcaster k8s.io/apimachinery@v0.20.2/pkg/watch/mux.go:73 +0xe4 goroutine 214 [chan receive]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x15fd600, 0x4000545ec0, 0x4000217e10) k8s.io/client-go@v0.20.2/tools/record/event.go:301 +0x98 created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher k8s.io/client-go@v0.20.2/tools/record/event.go:299 +0x64 goroutine 215 [chan receive]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x15fd600, 0x40002fc0c0, 0x40002fc060) k8s.io/client-go@v0.20.2/tools/record/event.go:301 +0x98 created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher k8s.io/client-go@v0.20.2/tools/record/event.go:299 +0x64 goroutine 216 [select]: github.com/eapache/channels.(*RingChannel).ringBuffer(0x4000218c00) github.com/eapache/channels@v1.1.0/ring_channel.go:87 +0x18c created by github.com/eapache/channels.NewRingChannel github.com/eapache/channels@v1.1.0/ring_channel.go:32 +0x18c goroutine 217 [syscall]: syscall.Syscall6(0x16, 0xa, 0x4000d65b40, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x0) syscall/asm_linux_arm64.s:35 +0x10 golang.org/x/sys/unix.EpollWait(0xa, 0x4000d65b40, 0x7, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0) golang.org/x/sys@v0.0.0-20210112080510-489259a85091/unix/zsyscall_linux_arm64.go:55 +0x58 github.com/fsnotify/fsnotify.(*fdPoller).wait(0x4000700d20, 0x0, 0x0, 0x0) github.com/fsnotify/fsnotify@v1.4.9/inotify_poller.go:86 +0x60 github.com/fsnotify/fsnotify.(*Watcher).readEvents(0x40002430e0) github.com/fsnotify/fsnotify@v1.4.9/inotify.go:192 +0x1a8 created by github.com/fsnotify/fsnotify.NewWatcher github.com/fsnotify/fsnotify@v1.4.9/inotify.go:59 +0x168 goroutine 218 [select]: k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch.func1(0x40002430e0, 0x4000623ec0, 0x40006305b0, 0xfffffcbdf8d2, 0x4) k8s.io/ingress-nginx/internal/watch/file_watcher.go:73 +0x90 created by k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch k8s.io/ingress-nginx/internal/watch/file_watcher.go:71 +0x148 goroutine 219 [syscall]: syscall.Syscall6(0x16, 0xe, 0x4000d85b40, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x0) syscall/asm_linux_arm64.s:35 +0x10 golang.org/x/sys/unix.EpollWait(0xe, 0x4000d85b40, 0x7, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0) golang.org/x/sys@v0.0.0-20210112080510-489259a85091/unix/zsyscall_linux_arm64.go:55 +0x58 github.com/fsnotify/fsnotify.(*fdPoller).wait(0x4000700f20, 0x0, 0x0, 0x0) github.com/fsnotify/fsnotify@v1.4.9/inotify_poller.go:86 +0x60 github.com/fsnotify/fsnotify.(*Watcher).readEvents(0x4000243130) github.com/fsnotify/fsnotify@v1.4.9/inotify.go:192 +0x1a8 created by github.com/fsnotify/fsnotify.NewWatcher github.com/fsnotify/fsnotify@v1.4.9/inotify.go:59 +0x168 goroutine 220 [select]: k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch.func1(0x4000243130, 0x4000623ee0, 0x40006305d0, 0xfffffcbdf908, 0x3) k8s.io/ingress-nginx/internal/watch/file_watcher.go:73 +0x90 created by k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch k8s.io/ingress-nginx/internal/watch/file_watcher.go:71 +0x148 goroutine 221 [chan receive]: k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0x4000bcb700) k8s.io/apimachinery@v0.20.2/pkg/watch/mux.go:219 +0x4c created by k8s.io/apimachinery/pkg/watch.NewBroadcaster k8s.io/apimachinery@v0.20.2/pkg/watch/mux.go:73 +0xe4 goroutine 222 [chan receive]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x15fd600, 0x400054a060, 0x4000630670) k8s.io/client-go@v0.20.2/tools/record/event.go:301 +0x98 created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher k8s.io/client-go@v0.20.2/tools/record/event.go:299 +0x64 goroutine 223 [chan receive]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x15fd600, 0x400054a390, 0x400054a360) k8s.io/client-go@v0.20.2/tools/record/event.go:301 +0x98 created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher k8s.io/client-go@v0.20.2/tools/record/event.go:299 +0x64 goroutine 225 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0x40003dca80) k8s.io/client-go@v0.20.2/util/workqueue/queue.go:198 +0x9c created by k8s.io/client-go/util/workqueue.newQueue k8s.io/client-go@v0.20.2/util/workqueue/queue.go:58 +0x134 goroutine 226 [select]: k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0x40003dcde0) k8s.io/client-go@v0.20.2/util/workqueue/delaying_queue.go:231 +0x324 created by k8s.io/client-go/util/workqueue.newDelayingQueue k8s.io/client-go@v0.20.2/util/workqueue/delaying_queue.go:68 +0x164 goroutine 227 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0x40003dcea0) k8s.io/client-go@v0.20.2/util/workqueue/queue.go:198 +0x9c created by k8s.io/client-go/util/workqueue.newQueue k8s.io/client-go@v0.20.2/util/workqueue/queue.go:58 +0x134 goroutine 228 [select]: k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0x40003dcfc0) k8s.io/client-go@v0.20.2/util/workqueue/delaying_queue.go:231 +0x324 created by k8s.io/client-go/util/workqueue.newDelayingQueue k8s.io/client-go@v0.20.2/util/workqueue/delaying_queue.go:68 +0x164 goroutine 241 [syscall]: syscall.Syscall6(0x16, 0x12, 0x4000e01b40, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0, 0x0, 0x0) syscall/asm_linux_arm64.s:35 +0x10 golang.org/x/sys/unix.EpollWait(0x12, 0x4000e01b40, 0x7, 0x7, 0xffffffffffffffff, 0x0, 0x0, 0x0) golang.org/x/sys@v0.0.0-20210112080510-489259a85091/unix/zsyscall_linux_arm64.go:55 +0x58 github.com/fsnotify/fsnotify.(*fdPoller).wait(0x400078ebe0, 0x0, 0x0, 0x0) github.com/fsnotify/fsnotify@v1.4.9/inotify_poller.go:86 +0x60 github.com/fsnotify/fsnotify.(*Watcher).readEvents(0x40001c60a0) github.com/fsnotify/fsnotify@v1.4.9/inotify.go:192 +0x1a8 created by github.com/fsnotify/fsnotify.NewWatcher github.com/fsnotify/fsnotify@v1.4.9/inotify.go:59 +0x168 goroutine 242 [select]: k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch.func1(0x40001c60a0, 0x4000b1f060, 0x4000bcfc60, 0x136b03f, 0xa) k8s.io/ingress-nginx/internal/watch/file_watcher.go:73 +0x90 created by k8s.io/ingress-nginx/internal/watch.(*OSFileWatcher).watch k8s.io/ingress-nginx/internal/watch/file_watcher.go:71 +0x148 [root@centos7 nginx_ingress]#
[root@centos7 nginx_ingress]# ls rootfs/etc/nginx fastcgi.conf fastcgi_params geoip koi-win mime.types modsecurity nginx.conf opentracing.json scgi_params template uwsgi_params.default fastcgi.conf.default fastcgi_params.default koi-utf lua mime.types.default modules nginx.conf.default owasp-modsecurity-crs scgi_params.default uwsgi_params win-utf [root@centos7 nginx_ingress]# ls rootfs2/etc/n network/ nginx/ [root@centos7 nginx_ingress]# ls rootfs2/etc/nginx/ conf.d fastcgi.conf fastcgi_params lua mime.types modules nginx.conf opentracing.json scgi_params secrets stream-conf.d template uwsgi_params [root@centos7 nginx_ingress]#
最终 方法
基础镜像有问题
用个全面的
BASE_IMAGE ?= willdockerhub/ingress-nginx-controller:v0.47.0
或者编译
https://github.com/kubernetes/ingress-nginx/blob/main/images/nginx/Makefile
[root@centos7 nginx_ingress]# kubectl logs ingress-nginx-controller-577fbcd469-gnv5f -n ingress-nginx ------------------------------------------------------------------------------- NGINX Ingress controller Release: v0.48.1 Build: git-f222c752b Repository: https://github.com/kubernetes/ingress-nginx.git nginx version: nginx/1.20.1 ------------------------------------------------------------------------------- I0805 06:36:11.435257 8 nginx.go:296] "Starting NGINX process" I0805 06:36:11.435262 8 leaderelection.go:243] attempting to acquire leader lease ingress-nginx/ingress-controller-leader-nginx... I0805 06:36:11.435895 8 nginx.go:316] "Starting validation webhook" address=":8443" certPath="/usr/local/certificates/cert" keyPath="/usr/local/certificates/key" I0805 06:36:11.436117 8 controller.go:148] "Configuration changes detected, backend reload required" I0805 06:36:11.442066 8 leaderelection.go:253] successfully acquired lease ingress-nginx/ingress-controller-leader-nginx I0805 06:36:11.442103 8 status.go:84] "New leader elected" identity="ingress-nginx-controller-577fbcd469-gnv5f" I0805 06:36:11.481209 8 controller.go:165] "Backend successfully reloaded" I0805 06:36:11.481283 8 controller.go:176] "Initial sync, sleeping for 1 second" I0805 06:36:11.481348 8 event.go:282] Event(v1.ObjectReference{Kind:"Pod", Namespace:"ingress-nginx", Name:"ingress-nginx-controller-577fbcd469-gnv5f", UID:"19a0a2fe-efb9-4d47-90a6-1467a95a9684", APIVersion:"v1", ResourceVersion:"8239247", FieldPath:""}): type: 'Normal' reason: 'RELOAD' NGINX reload triggered due to a change in configuration 2021/08/05 06:36:11 [alert] 3904#3904: pthread_create() failed (11: Resource temporarily unavailable) 2021/08/05 06:36:11 [alert] 3823#3823: pthread_create() failed (11: Resource temporarily unavailable) 2021/08/05 06:36:11 [alert] 4016#4016: pthread_create() failed (11: Resource temporarily unavailable) 2021/08/05 06:36:11 [alert] 36#36: fork() failed while spawning "cache manager process" (11: Resource temporarily unavailable) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor) 2021/08/05 06:36:11 [alert] 36#36: sendmsg() failed (9: Bad file descriptor)
mandatory.yaml
wget --no-check-certificate https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml[root@centos7 nginx_ingress]# kubectl logs nginx-ingress-controller-566cfc54bc-2nkhw -n ingress-nginx ------------------------------------------------------------------------------- NGINX Ingress controller Release: v0.48.1 Build: git-f222c752b Repository: https://github.com/kubernetes/ingress-nginx.git nginx version: nginx/1.21.0 ------------------------------------------------------------------------------- W0805 03:34:46.935667 8 client_config.go:614] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work. I0805 03:34:46.936075 8 main.go:241] "Creating API client" host="https://10.96.0.1:443" I0805 03:34:46.949839 8 main.go:285] "Running in Kubernetes cluster" major="1" minor="18" git="v1.18.0" state="clean" commit="9e991415386e4cf155a24b1da15becaa390438d8" platform="linux/arm64" F0805 03:34:46.953496 8 main.go:93] No service with name ingress-nginx found in namespace ingress-nginx: services "ingress-nginx" not found goroutine 1 [running]: k8s.io/klog/v2.stacks(0x400000e001, 0x400052e0f0, 0x92, 0xe2) k8s.io/klog/v2@v2.4.0/klog.go:1026 +0xac k8s.io/klog/v2.(*loggingT).output(0x227f360, 0x4000000003, 0x0, 0x0, 0x40002b1f10, 0x21cd0e7, 0x7, 0x5d, 0x4000241e00) k8s.io/klog/v2@v2.4.0/klog.go:975 +0x17c k8s.io/klog/v2.(*loggingT).printDepth(0x227f360, 0x4000000003, 0x0, 0x0, 0x0, 0x0, 0x1, 0x400030ecd0, 0x1, 0x1) k8s.io/klog/v2@v2.4.0/klog.go:732 +0x120 k8s.io/klog/v2.(*loggingT).print(...) k8s.io/klog/v2@v2.4.0/klog.go:714 k8s.io/klog/v2.Fatal(...) k8s.io/klog/v2@v2.4.0/klog.go:1482 main.main() k8s.io/ingress-nginx/cmd/nginx/main.go:93 +0x14f0 goroutine 6 [chan receive]: k8s.io/klog/v2.(*loggingT).flushDaemon(0x227f360) k8s.io/klog/v2@v2.4.0/klog.go:1169 +0x70 created by k8s.io/klog/v2.init.0 k8s.io/klog/v2@v2.4.0/klog.go:417 +0x124 goroutine 179 [IO wait]: internal/poll.runtime_pollWait(0xffff6d234fc0, 0x72, 0xffffffffffffffff) runtime/netpoll.go:203 +0x44 internal/poll.(*pollDesc).wait(0x40002e5118, 0x72, 0x800, 0x8b9, 0xffffffffffffffff) internal/poll/fd_poll_runtime.go:87 +0x38 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0x40002e5100, 0x4000292900, 0x8b9, 0x8b9, 0x0, 0x0, 0x0) internal/poll/fd_unix.go:169 +0x158 net.(*netFD).Read(0x40002e5100, 0x4000292900, 0x8b9, 0x8b9, 0x0, 0x8b4, 0x4000241908) net/fd_unix.go:202 +0x44 net.(*conn).Read(0x400021a018, 0x4000292900, 0x8b9, 0x8b9, 0x0, 0x0, 0x0) net/net.go:184 +0x74 crypto/tls.(*atLeastReader).Read(0x40003f31a0, 0x4000292900, 0x8b9, 0x8b9, 0xe4, 0x8b4, 0x4000241938) crypto/tls/conn.go:760 +0x58 bytes.(*Buffer).ReadFrom(0x400029a258, 0x15eeba0, 0x40003f31a0, 0x2d29f4, 0x1166420, 0x12d8440) bytes/buffer.go:204 +0xa4 crypto/tls.(*Conn).readFromUntil(0x400029a000, 0x15f0e60, 0x400021a018, 0x5, 0x400021a018, 0xd3) crypto/tls/conn.go:782 +0xd0 crypto/tls.(*Conn).readRecordOrCCS(0x400029a000, 0x0, 0x0, 0x152b8) crypto/tls/conn.go:589 +0xd8 crypto/tls.(*Conn).readRecord(...) crypto/tls/conn.go:557 crypto/tls.(*Conn).Read(0x400029a000, 0x40005fb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) crypto/tls/conn.go:1233 +0x11c bufio.(*Reader).Read(0x4000429560, 0x40005f2118, 0x9, 0x9, 0x40001f3a00, 0x494a18, 0x40001f3a01) bufio/bufio.go:226 +0x238 io.ReadAtLeast(0x15eea00, 0x4000429560, 0x40005f2118, 0x9, 0x9, 0x9, 0x40001a8060, 0x0, 0x15eed60) io/io.go:310 +0x78 io.ReadFull(...) io/io.go:329 golang.org/x/net/http2.readFrameHeader(0x40005f2118, 0x9, 0x9, 0x15eea00, 0x4000429560, 0x0, 0x4000000000, 0xcb6fc, 0x40002185f8) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:237 +0x5c golang.org/x/net/http2.(*Framer).ReadFrame(0x40005f20e0, 0x400002f920, 0x0, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/frame.go:492 +0x74 golang.org/x/net/http2.(*clientConnReadLoop).run(0x4000241fa8, 0x0, 0x0) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1819 +0xb0 golang.org/x/net/http2.(*ClientConn).readLoop(0x40004c4c00) golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:1741 +0x60 created by golang.org/x/net/http2.(*Transport).newClientConn golang.org/x/net@v0.0.0-20201110031124-69a78807bb2b/http2/transport.go:705 +0x570 [root@centos7 nginx_ingress]#
go build
docker export
rootfs/Dockerfile
RUN apk add --no-cache libcap && setcap cap_net_bind_service=+ep /nginx-ingress-controller && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller && setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx && apk del libcap
docker export 8c2f01fdd438 | tar -C rootfs -xvf -
docker export 8c2f01fdd438 | tar -C rootfs -xvf -
docker export 8c2f01fdd438 | tar -C rootfs -xvf -
[root@centos7 rootfs]# ls bin dbg dev etc home ingress-controller lib media mnt nginx-ingress-controller opt proc root run sbin srv sys tmp usr var wait-shutdown [root@centos7 rootfs]# ls usr/ bin lib local sbin share [root@centos7 rootfs]# ls usr/local/nginx/ html sbin [root@centos7 rootfs]# ls usr/local/nginx/sbin/ nginx [root@centos7 rootfs]# usr/local/nginx/sbin/nginx -v -bash: usr/local/nginx/sbin/nginx: /lib/ld-musl-aarch64.so.1: bad ELF interpreter: No such file or directory [root@centos7 rootfs]#
make build
export 一个官方镜像
[root@centos7 rootfs]# cat etc/os-release NAME="Alpine Linux" ID=alpine VERSION_ID=3.13.5 PRETTY_NAME="Alpine Linux v3.13" HOME_URL="https://alpinelinux.org/" BUG_REPORT_URL="https://bugs.alpinelinux.org/"
BASE_IMAGE ?= nginx/nginx-ingress:1.12.0-alpine
build/run-in-docker.sh
E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v08042018-e2f5d90
改成
E2E_IMAGE=docker.io/willdockerhub/ingress-nginx-controller:v0.47.0
E2E_IMAGE=docker.io/willdockerhub/ingress-nginx-controller:v0.47.0
#E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v08042018-e2f5d90
#E2E_IMAGE=${E2E_IMAGE:-k8s.gcr.io/ingress-nginx/e2e-test-runner:v20210601-g96a87c79b@sha256:f84dcddc84e5cba220260f315e18cd47fc8c6b7f3f4f57b7b3e9cc2ea25324b7}
#E2E_IMAGE=${E2E_IMAGE:-nginx/nginx-ingress:1.12.0-alpine}
root@ubuntu:~/nginx_ingress/ingress-nginx# make build build/build.sh: line 49: go: command not found Makefile:80: recipe for target 'build' failed make: *** [build] Error 127
# create output directory as current user to avoid problem with docker. mkdir -p "${KUBE_ROOT}/bin" "${KUBE_ROOT}/bin/${ARCH}" if [[ "$DOCKER_IN_DOCKER_ENABLED" == "true" ]]; then /bin/bash -c "${FLAGS}" else docker run --tty --rm ${DOCKER_OPTS} -e GOCACHE="/go/src/${PKG}/.cache" -e DOCKER_IN_DOCKER_ENABLED="true" -v "${HOME}/.kube:${HOME}/.kube" -v "${KUBE_ROOT}:/go/src/${PKG}" -v "${KUBE_ROOT}/bin/${ARCH}:/go/bin/linux_${ARCH}" -v "/var/run/docker.sock:/var/run/docker.sock" -v "${INGRESS_VOLUME}:/etc/ingress-controller/" -w "/go/src/${PKG}" -u $(id -u ${USER}):$(id -g ${USER}) ${E2E_IMAGE} /bin/bash -c "${FLAGS}" fi root@ubuntu:~/nginx_ingress/ingress-nginx# env DOCKER_IN_DOCKER_ENABLED="f" make build build/build.sh: line 49: go: command not found Makefile:80: recipe for target 'build' failed make: *** [build] Error 127 root@ubuntu:~/nginx_ingress/ingress-nginx# env DOCKER_IN_DOCKER_ENABLED="true" make build go: downloading k8s.io/api v0.20.2 go: downloading github.com/prometheus/client_golang v1.7.1 go: downloading k8s.io/apimachinery v0.20.2 go: downloading k8s.io/apiserver v0.20.2 go: downloading k8s.io/client-go v0.20.2 go: downloading github.com/cespare/xxhash v1.1.0 go: downloading github.com/eapache/channels v1.1.0 go: downloading gopkg.in/go-playground/pool.v3 v3.1.1 go: downloading github.com/cespare/xxhash/v2 v2.1.1 go: downloading github.com/gogo/protobuf v1.3.1 go: downloading github.com/prometheus/client_model v0.2.0 go: downloading github.com/ncabatoff/process-exporter v0.7.2 go: downloading pault.ag/go/sniff v0.0.0-20200207005214-cf7e4d167732
生成文件
env DOCKER_IN_DOCKER_ENABLED="true" make build
root@ubuntu:~/nginx_ingress/ingress-nginx/build# vi build.sh +54 #!/bin/bash # Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. if [ -n "$DEBUG" ]; then set -x fi set -o errexit set -o nounset set -o pipefail declare -a mandatory mandatory=( PKG ARCH COMMIT_SHA REPO_INFO TAG ) missing=false for var in "${mandatory[@]}"; do if [[ -z "${!var:-}" ]]; then echo "Environment variable $var must be set" missing=true fi done if [ "$missing" = true ]; then exit 1 fi export CGO_ENABLED=0 export GOARCH=${ARCH} go build -trimpath -ldflags="-buildid= -w -s -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT_SHA} -X ${PKG}/version.REPO=${REPO_INFO}" -o "rootfs/bin/${ARCH}/nginx-ingress-controller" "${PKG}/cmd/nginx" go build -trimpath -ldflags="-buildid= -w -s -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT_SHA} -X ${PKG}/version.REPO=${REPO_INFO}" -o "rootfs/bin/${ARCH}/dbg" "${PKG}/cmd/dbg" go build -trimpath -ldflags="-buildid= -w -s -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT_SHA} -X ${PKG}/version.REPO=${REPO_INFO}" root@ubuntu:~/nginx_ingress/ingress-nginx/build# ls ../rootfs/bin/arm64/ dbg nginx-ingress-controller wait-shutdown
dbg
[root@centos7 rootfs]# ./dbg dbg is a tool for quickly inspecting the state of the nginx instance Usage: dbg [command] Available Commands: backends Inspect the dynamically-loaded backends information certs Inspect dynamic SSL certificates conf Dump the contents of /etc/nginx/nginx.conf general Output the general dynamic lua state help Help about any command Flags: -h, --help help for dbg --status-port int Port to use for the lua HTTP endpoint configuration. (default 10246) Use "dbg [command] --help" for more information about a command.
定制nginx
https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx
bug
root@ubuntu:~/nginx_ingress# kubectl create -f ingress.yaml Error from server (InternalError): error when creating "ingress.yaml": Internal error occurred: failed calling webhook "validate.nginx.ingress.kubernetes.io": Post https://ingress-nginx-controller-admission.ingress-nginx.svc:443/networking/v1beta1/ingresses?timeout=10s: dial tcp 10.97.5.227:443: connect: connection refused root@ubuntu:~/nginx_ingress# kubectl get pods --all-namespaces -o wide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES default busybox 1/1 Running 146 6d2h 10.244.129.145 centos7 <none> <none> default example-foo-54dc4db9fc-42jz6 1/1 Running 0 5d1h 10.244.129.160 centos7 <none> <none> default web2-7cdf5dffb-26xrn 1/1 Running 0 5h34m 10.244.41.1 cloud <none> <none> default web3-c9654466d-xwb5j 1/1 Running 0 5h33m 10.244.41.2 cloud <none> <none> ingress-nginx ingress-nginx-admission-create-9qnqf 0/1 Completed 0 37m 10.244.129.147 centos7 <none> <none> ingress-nginx ingress-nginx-admission-patch-dh6z5 0/1 Completed 2 37m 10.244.129.146 centos7 <none> <none> ingress-nginx ingress-nginx-controller-687ff7d857-2g5w7 0/1 CrashLoopBackOff 2 37m 10.244.129.148 centos7 <none> <none>