基于阿里云容器镜像服务加速K8S镜像下载(二)_阿里云镜像下载calico

简单说明

• 在《基于阿里云容器镜像服务加速K8S镜像下载》中介绍了使用阿里云镜像服务来解决一些不可达镜像的下载
• 但是需要反复在阿里云容器镜像控制台添加仓库，这种操作实在太傻，是电脑这头蠢驴该干的事
• 在阿里云官网找到了镜像服务相关的API文档，但是使用方法过于含糊
• 最终找到了一篇播客 https://blog.csdn.net/catoop/article/details/90903890
• 受该播客启发，最终形成了获取访问token，使用docker registry v2 api的方法的自动化批量镜像加速方案

具体部署步骤

• 打开https://code.aliyun.com，添加一个新项目 docker-auto
• 打开阿里云镜像服务，创建一个新的命名空间 vinc-auto，打开自动创建仓库的功能，并设置为公有仓库
• 创建一个镜像仓库，image-auto，关联docker-auto，暂时不添加构建规则
• 找一台能够上公网的Linux机器，依据《CentOS7部署安装Docker和Docker Compose工具简录》部署docker环境，开始我们的骚操作

ssh-keygen
ssh-copy-id -o StrictHostKeyChecking=no 127.0.0.1
cat ~/.ssh/id_rsa.pub
# 将公钥添加到https://code.aliyun.com对应的代码库中
1
2
3
4

cd /tmp
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache faster
yum -y install jq git
git config --global user.name  "zwjqb"
git config --global user.email "zwjqb@126.com"
git clone git@code.aliyun.com:zwjqb/docker-auto.git
cd docker-auto
echo "FROM busybox">Dockerfile
git add .
git commit -m "new image: busybox"
git push -u origin master
# 此时可以打开镜像仓库，添加构建规则，并立即构建
1
2
3
4
5
6
7
8
9
10
11
12
13

# 镜像的hashvalue检测脚本
cat >/tmp/image-hash.sh<<EOF
#!/bin/bash
source ~/.bash_profile
User=\$1
Pass=\$2
ImageUrl=\$3
Repons=\$4
Repo=\$5
Key=Www-Authenticate
Content=\$(curl -s --head -X GET https://\${ImageUrl}/v2/\${Repons}/\${Repo}/manifests/latest|grep \${Key})
Realm=\$(echo \${Content}|awk -F'"' '{print \$2}')
Service=\$(echo \${Content}|awk -F'"' '{print \$4}')
Scope=\$(echo \${Content}|awk -F'"' '{print \$6}')
Token=\$(curl -s -u clientId:clientSecret -X POST \
  -d "username=\${User}\&password=\${Pass}&service=\${Service}&scope=\${Scope}" \
  "\${Realm}"|jq '.access_token'|sed 's/"//g')
# Repolist=\$(curl -s -u clientId:clientSecret \
#   -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
#   -H "Authorization: Bearer \${Token}" https://\${ImageUrl}/v2/\${Repons}/\${Repo}/tags/list)
# ImageName=\$(echo \${Repolist}|jq '.name'|sed 's/"//g')
# ImageTags=\$(echo \${Repolist}|jq '.tags[]'|sed 's/"//g'|awk '{printf \$0" "}')
ImageManifests=\$(curl -s -u clientId:clientSecret \
  -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
  -H "Authorization: Bearer \${Token}" https://\${ImageUrl}/v2/\${Repons}/\${Repo}/manifests/latest)
ImageHashValue=\$(echo \${ImageManifests}|jq '.config.digest'|sed 's/"//g')
echo \${ImageHashValue}
EOF
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 更新git仓库，自动触发构建，下载修改标签并回传的脚本
# 如果半小时内没有构建成功，则重新触发最多两次构建，两次后跨过该镜像的构建
cat >/tmp/image.sh<<EOF
#!/bin/bash
source ~/.bash_profile
cd /tmp/docker-auto
Log=/tmp/image.log
Username=\$1
Userpass=\$2
ImageUrl=\$3
Repons=\$4
Repo=\$5
echo ''>\${Log}
ImageHashValue=\$(bash /tmp/image-hash.sh \${Username} \${Userpass} \${ImageUrl} \${Repons} \${Repo})
cat /tmp/image-list.txt|while read line
do
    Name=\$(basename \$(echo \${line}|awk -F':' '{print \$1}'))
    Tag=\$(echo \${line}|awk -F':' '{print \$2}')
    for i in \$(seq 180)
    do
        if [ \$((\${i}%60)) -eq 1 ]
        then
            if [ \${i} -eq 1 ]
            then
                echo "[+] \$(date +%F_%T) 开始构建镜像: \${line}">>\${Log}
                echo "FROM \${line}">Dockerfile
            else
                echo "  [-] \$(date +%F_%T) 构建失败,重新触发构建镜像: \${line}">>\${Log}
                echo "# [-] \$(date +%F_%T) 构建失败,重新触发构建镜像: \${line} \${i}">>Dockerfile
            fi
            git add .
            git commit -m "new image: \${line}"
            git push -u origin master
        else
            if [ \${ImageHashValue} == "\$(bash /tmp/image-hash.sh \${Username} \${Userpass} \${ImageUrl} \${Repons} \${Repo})" ]
            then
                echo "  [-] \$(date +%F_%T) 等待镜像构建...">>\${Log}
                sleep 30
            else
                ImageHashValue=\$(bash /tmp/image-hash.sh \${Username} \${Userpass} \${ImageUrl} \${Repons} \${Repo})
                docker pull \${ImageUrl}/\${Repons}/\${Repo}
                docker tag  \${ImageUrl}/\${Repons}/\${Repo} \${ImageUrl}/\${Repons}/\${Name}:\${Tag}
                docker push \${ImageUrl}/\${Repons}/\${Name}:\${Tag}
                docker rmi  \${ImageUrl}/\${Repons}/\${Repo}
                docker rmi  \${ImageUrl}/\${Repons}/\${Name}:\${Tag}
                echo "  [-] \$(date +%F_%T) 构建回传完成 \${ImageUrl}/\${Repons}/\${Name}:\${Tag}">>\${Log}
                break
            fi
        fi
    done
done
EOF
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

# 创建需要的镜像列表
cat >/tmp/image-list.txt<<EOF
gcr.io/google-containers/addon-resizer:1.8.3
gcr.io/google-containers/cluster-proportional-autoscaler-amd64:1.6.0
gcr.io/google-containers/k8s-dns-node-cache:1.15.8
gcr.io/google_containers/kube-registry-proxy:0.4
gcr.io/google_containers/kubernetes-dashboard-amd64:v1.10.1
gcr.io/google_containers/metrics-server-amd64:v0.3.3
gcr.io/google_containers/pause-amd64:3.1
gcr.io/kubernetes-helm/tiller:v2.16.1
quay.io/coreos/etcd:v3.3.10
quay.io/coreos/flannel-cni:v0.3.0
quay.io/coreos/flannel:v0.11.0
quay.io/external_storage/cephfs-provisioner:v2.1.0-k8s1.11
quay.io/external_storage/local-volume-provisioner:v2.3.2
quay.io/external_storage/rbd-provisioner:v2.1.1-k8s1.11
quay.io/jetstack/cert-manager-controller:v0.11.0
quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
quay.io/l23network/k8s-netchecker-agent:v1.0
quay.io/l23network/k8s-netchecker-server:v1.0
docker.io/calico/cni:v3.7.3
docker.io/calico/kube-controllers:v3.7.3
docker.io/calico/node:v3.7.3
docker.io/calico/pod2daemon-flexvol:v3.13.0
docker.io/calico/routereflector:v0.6.1
docker.io/calico/typha:v3.7.3
EOF
# 注意，镜像名字和版本不能重复，否则会被覆盖
cat /tmp/image-list.txt|awk -F'/' '{print $NF}'|sort|uniq -c|awk '{if($1!=1)print $0}'
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# 设置阿里云镜像服务的用户名和密码，进行登陆验证，最终执行脚本
Username=zwjqb@126.com
Userpass=**********
ImageUrl=registry.cn-hangzhou.aliyuncs.com
Repons=vinc-auto
Repo=image-auto
docker login --username=${Username} ${ImageUrl}
bash /tmp/image.sh ${Username} ${Userpass} ${ImageUrl} ${Repons} ${Repo} &
1
2
3
4
5
6
7
8

[TOC]