harbor基于keepalive高可用部署

Posted 2020-12-10

6.Harbor配置
6.1.harbor01节点
1.修改harbor.cfg
cat >/root/harbor/harbor.cfg <<-‘EOF‘
_version = 1.5.0
hostname = reg.xgmin.com
ui_url_protocol = https
max_job_workers = 50
customize_crt = off
ssl_cert = /data/cert/reg.xgmin.com.crt
ssl_cert_key = /data/cert/reg.xgmin.com.key
secretkey_path = /data
admiral_url = NA
log_rotate_count = 50
log_rotate_size = 200M

http_proxy =
https_proxy =
no_proxy = 127.0.0.1,localhost,ui

email_identity =
email_server = smtp.mydomain.com
email_server_port = 25
email_username = [e-mail]
email_password = abc
email_from = admin <[e-mail]>
email_ssl = false
email_insecure = false

harbor_admin_password = Harbor12345

auth_mode = db_auth
ldap_url = ldaps://ldap.mydomain.com
ldap_basedn = ou=people,dc=mydomain,dc=com
ldap_uid = uid
ldap_scope = 2
ldap_timeout = 5
ldap_verify_cert = true
ldap_group_basedn = ou=group,dc=mydomain,dc=com
ldap_group_filter = objectclass=group
ldap_group_gid = cn
ldap_group_scope = 2
self_registration = off
token_expiration = 30
project_creation_restriction = everyone

db_host = 10.7.132.243
db_password = Wab1IJvdHurMbPUp
db_port = 3306
db_user = root

redis_url = 10.7.132.243:6379

clair_db_host = 10.7.132.243
clair_db_password = bXTCUL5BIz5a4liM
clair_db_port = 5432
clair_db_username = postgres
clair_db = postgres

uaa_endpoint = uaa.mydomain.org
uaa_clientid = id
uaa_clientsecret = secret
uaa_verify_cert = true
uaa_ca_cert = /path/to/ca.pem

#registry_storage_provider_name = filesystem
#registry_storage_provider_config =

registry_storage_provider_name = s3
registry_storage_provider_config = accesskey: NCGOJZXAHDJIIDBYUFKD,secretkey: c8d0v3ENh5ZlgSOMjd0oaLvZZSdITjkjDsmwKxbS,region: yzqsp1,regionendpoint: http://s3.yzqsp1.stor.qycloud.com,bucket: ghqharbortest,secure: false
EOF
2.拷贝证书到/etc/docker和/data/cert
mkdir -p /data/cert
cp /root/cert/reg.xgmin.com.crt /root/cert/reg.xgmin.com.key /data/cert/

cd /root/harbor/ && ./prepare --ha
mkdir -p /etc/docker/certs.d/reg.xgmin.com
cp /root/cert/reg.xgmin.com.crt /etc/docker/certs.d/reg.xgmin.com/
ll /etc/docker/certs.d/reg.xgmin.com/

systemctl restart docker
3.执行安装
./install.sh --ha
4.出现下方提示后，浏览器访问https://10.7.132.243
? ----Harbor has been installed and started successfully.----

Now you should be able to visit the admin portal at https://reg.xgmin.com.
For more details, please visit https://github.com/vmware/harbor .
5.输入默认用户名密码登录成功后在操作keepalived

6.2.harbor02节点
1.修改harbor.cfg
cat >/root/harbor/harbor.cfg <<-‘EOF‘
_version = 1.5.0
hostname = reg.xgmin.com
ui_url_protocol = https
max_job_workers = 50
customize_crt = off
ssl_cert = /data/cert/reg.xgmin.com.crt
ssl_cert_key = /data/cert/reg.xgmin.com.key
secretkey_path = /data
admiral_url = NA
log_rotate_count = 50
log_rotate_size = 200M

http_proxy =
https_proxy =
no_proxy = 127.0.0.1,localhost,ui

email_identity =
email_server = smtp.mydomain.com
email_server_port = 25
email_username = [e-mail]
email_password = abc
email_from = admin <[e-mail]>
email_ssl = false
email_insecure = false

harbor_admin_password = Harbor12345

auth_mode = db_auth
ldap_url = ldaps://ldap.mydomain.com
ldap_basedn = ou=people,dc=mydomain,dc=com
ldap_uid = uid
ldap_scope = 2
ldap_timeout = 5
ldap_verify_cert = true
ldap_group_basedn = ou=group,dc=mydomain,dc=com
ldap_group_filter = objectclass=group
ldap_group_gid = cn
ldap_group_scope = 2
self_registration = off
token_expiration = 30
project_creation_restriction = everyone

db_host = 10.7.132.243
db_password = Wab1IJvdHurMbPUp
db_port = 3306
db_user = root

redis_url = 10.7.132.243:6379

clair_db_host = 10.7.132.243
clair_db_password = bXTCUL5BIz5a4liM
clair_db_port = 5432
clair_db_username = postgres
clair_db = postgres

uaa_endpoint = uaa.mydomain.org
uaa_clientid = id
uaa_clientsecret = secret
uaa_verify_cert = true
uaa_ca_cert = /path/to/ca.pem

#registry_storage_provider_name = filesystem
#registry_storage_provider_config =

registry_storage_provider_name = s3
registry_storage_provider_config = accesskey: NCGOJZXAHDJIIDBYUFKD,secretkey: c8d0v3ENh5ZlgSOMjd0oaLvZZSdITjkjDsmwKxbS,region: yzqsp1,regionendpoint: http://s3.yzqsp1.stor.qycloud.com,bucket: ghqharbortest,secure: false
EOF
2.拷贝证书到/etc/docker和/data/cert
mkdir -p /data/cert
cp /root/cert/reg.xgmin.com.crt /root/cert/reg.xgmin.com.key /data/cert/

cd /root/harbor/ && ./prepare --ha
mkdir -p /etc/docker/certs.d/reg.xgmin.com
cp /root/cert/reg.xgmin.com.crt /etc/docker/certs.d/reg.xgmin.com/
ll /etc/docker/certs.d/reg.xgmin.com/

systemctl restart docker
3.执行安装
./install.sh --ha
4.出现下方提示后，浏览器访问https://10.7.132.219
? ----Harbor has been installed and started successfully.----

Now you should be able to visit the admin portal at https://reg.xgmin.com.
For more details, please visit https://github.com/vmware/harbor .
5.输入默认用户名密码登录成功后在操作keepalived
7.Keepalived配置
7.1.Master节点设置
1.在harbor01上写入keepalived的master配置文件
yum install keepalived -y
tee > /etc/keepalived/keepalived.conf <<-‘EOF‘
global_defs {
router_id harbar_ha
}

vrrp_script chk_nginx_proxy {
script "/etc/keepalived/scripts/nginx_check.sh"
interval 2
weight -20
}

vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 67
priority 151
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 431953
}
virtual_ipaddress {
10.7.132.253/24
}
track_script {
chk_nginx_proxy
}
}
EOF
2.增加keepalived检查脚本
mkdir -p /etc/keepalived/scripts/
tee > /etc/keepalived/scripts/nginx_check.sh <<-‘EOF‘
#!/bin/bash
nginxpid=ps -C nginx --no-header | wc -l
if [ $nginxpid -eq 0 ];then
systemctl stop keepalived
fi
EOF

chmod +x /etc/keepalived/scripts/nginx_check.sh
3.启动keepalived，
systemctl restart keepalived
systemctl enable keepalived
systemctl status keepalived
7.2.Backup节点设置
1.在harbor02上写入keepalived的backup配置文件
yum install keepalived -y
tee > /etc/keepalived/keepalived.conf <<-‘EOF‘
global_defs {
router_id harbar_ha
}

vrrp_script chk_nginx_proxy {
script "/etc/keepalived/scripts/nginx_check.sh"
interval 2
weight -20
}

vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 67
priority 101
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 431953
}
virtual_ipaddress {
10.7.132.253/24
}
track_script {
chk_nginx_proxy
}
}
EOF

2.增加keepalived检查脚本

mkdir -p /etc/keepalived/scripts/
tee > /etc/keepalived/scripts/nginx_check.sh <<-‘EOF‘
#!/bin/bash
nginxpid=ps -C nginx --no-header | wc -l
if [ $nginxpid -eq 0 ];then
systemctl stop keepalived
fi
EOF

chmod +x /etc/keepalived/scripts/nginx_check.sh
3.启动keepalived，
systemctl restart keepalived
systemctl enable keepalived
systemctl status keepalived
7.3.检查keepalived状态
1.harbor01节点查看
[root@harbor01 ~]# ip a | grep eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 10.7.132.243/24 brd 10.7.132.255 scope global dynamic eth0
inet 10.7.132.253/24 scope global secondary eth0:vip
1.harbor02节点查看
[root@harbor02 ~]# ip a | grep eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 10.7.132.219/24 brd 10.7.132.255 scope global dynamic eth0

8.检查测试
8.1.push镜像
1.到harbor01节点登录docker镜像仓库
[root@harbor01 ~]# docker login reg.xgmin.com
Username: admin
Password: Harbor12345
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

2.打tage并提交测试

[root@harbor01 ~]# docker tag photon:1.0 reg.xgmin.com/library/xgmintest:v1
[root@harbor01 ~]# docker push reg.xgmin.com/library/xgmintest:v1
The push refers to repository [reg.xgmin.com/library/xgmintest]
ad50e89f4922: Pushed
v1: digest: sha256:2336c23b341da8853d48f5e9234c1f3fa914db2acc773996fb0fbde33e57bb1c size: 529
8.2.web检查
登录reg.xgmin.com 查看我们上传的镜像

8.3.s3fs检查
此测试方式，是查看镜像是否存储到青云的s3对象存储上
1.安装s3fs
yum install s3fs-fuse -y
2.配置s3fs秘钥
mkdir -p /root/.s3fs/
cat > /root/.s3fs/credentials <<-‘EOF‘
NCGOJZXAHDJIIDBYUFKD:c8d0v3ENh5ZlgSOMjd0oaLvZZSdITjkjDsmwKxbS
EOF
chmod 600 /root/.s3fs/credentials
3.挂载s3fs
mkdir -p /mnt/mybucket-test
s3fs ghqharbortest /mnt/mybucket-test -o passwd_file=/root/.s3fs/credentials -o url=http://s3.yzqsp1.stor.qycloud.com
df -T | grep s3fs
4.查看上传的镜像
[root@harbor01 ~]# cd /mnt/mybucket-test/docker/
[root@harbor01 docker]# ll
总用量 1
drwxr-x--- 1 root root 0 1月 1 1970 registry
[root@harbor01 docker]# du -sh *
120M registry
[root@harbor01 docker]# tree
.
└── registry
└── v2
├── blobs
│?? └── sha256
│?? ├── 03
│?? │?? └── 03c1901c3cd5f7adfb65adaaee73428532a9571b794e17ef1677da667f80b1b5
│?? │?? └── data
│?? ├── 0d
│?? │?? └── 0dbcca2a156e7892be1414f91bac289595fdf210cebe315f733d72720efa89c1
│?? │?? └── data
│?? ├── 13
│?? │?? └── 13ae381fcfc572185c3ff094419c15ce493965a009e0997448d0214b0354cd47
│?? │?? └── data
│?? ├── 18
│?? │?? └── 18ceb72f6a2dbae1371887defb26620fd28ac989ec567a4d584ef965ee60eb52
│?? │?? └── data
│?? ├── 1f
│?? │?? └── 1fe4320e9ed89b03f0b3158a4336ceb08fcb44d949d84522b9688089617096ff
│?? │?? └── data
│?? ├── 23
│?? │?? └── 2336c23b341da8853d48f5e9234c1f3fa914db2acc773996fb0fbde33e57bb1c
│?? │?? └── data
8.4.客户端测试
1.找一台新的docker客户端进行拉取镜像测试，首先安装docker
yum install docker-ce -y
2.增加docker配置
mkdir -p /etc/docker
tee > /etc/docker/daemon.json <<-‘EOF‘
{
"registry-mirrors": ["https://reg.xgmin.com"]
}
EOF

cat >> /etc/hosts <<-‘EOF‘
10.7.132.253 reg.xgmin.com
EOF
3.配置docker使用指定目录和免https认证
#vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd --graph=/app/docker --storage-driver=overlay --insecure-registry=reg.xgmin.com
4.重启docker
systemctl daemon-reload && systemctl restart docker
5.拉取镜像
[root@i-7qd2o33x ~]# docker pull reg.xgmin.com/library/xgmintest:v1
v1: Pulling from library/xgmintest
5efd2aef02cd: Pull complete
Digest: sha256:2336c23b341da8853d48f5e9234c1f3fa914db2acc773996fb0fbde33e57bb1c
Status: Downloaded newer image for reg.xgmin.com/library/xgmintest:v1