kubeedge: keadm 源码学习

Posted 2022-11-28 Kris_u

一、keadm beta init provides a solution for integrating Cloudcore Helm Chart.

keadm beta init --advertise-address=$ip  --kubeedge-version=1.10.0 --kube-config=/root/.kube/config --force --set cloudCore.modules.dynamicController.enable=true

 keadm beta init 使用helm 安装cloudcore.

/kubeedge/manifests/charts/cloudcore/README.md 可以查看cloucore 的Custom Values

example installation:

helm upgrade --install cloudcore ./cloudcore --namespace kubeedge \\
--create-namespace -f ./cloudcore/values.yaml --set \\ 
cloudCore.modules.cloudHub.advertiseAddress[0]=192.168.88.6

keadm beta init 通过内置文件/kubeedge/manifests/charts/cloudcore/values.yaml 完成cloudcore的基本配置。

Usage:
  keadm beta init [flags]

Examples:

keadm beta init
- This command will render and install the Charts for Kubeedge cloud component

keadm beta init --advertise-address=127.0.0.1 --profile version=v1.9.0 --kube-config=/root/.kube/config
  - kube-config is the absolute path of kubeconfig which used to secure connectivity between cloudcore and kube-apiserver
	- a list of helm style set flags like "--set key=value" can be implemented, ref: https://github.com/kubeedge/kubeedge/tree/master/manifests/charts/cloudcore/README.md


Flags:
      --advertise-address string    Use this key to set IPs in cloudcore's certificate SubAltNames field. eg: 10.10.102.78,10.10.102.79
  -d, --dry-run                     Print the generated k8s resources on the stdout, not actual excute. Always use in debug mode
      --external-helm-root string   Add external helm root path to keadm.
  -f, --files string                Allow appending file directories of k8s resources to keadm, separated by commas
      --force                       Forced installing the cloud components without waiting.
  -h, --help                        help for init
      --kube-config string          Use this key to set kube-config path, eg: $HOME/.kube/config (default "/root/.kube/config")
      --kubeedge-version string     Use this key to set the default image tag
      --manifests string            Allow appending file directories of k8s resources to keadm, separated by commas
      --profile string              Set profile on the command line (iptablesMgrMode=external or version=v1.9.1)
      --set stringArray             Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)

函数入口：func NewInitBeta() *cobra.Command 
1、checkFlags 2、AddInitBeta2ToolsList 3、ExecuteInitBeta    

func NewInitBeta() *cobra.Command 
	BetaInit := newInitBetaOptions()

	tools := make(map[string]types.ToolsInstaller)
	flagVals := make(map[string]types.FlagData)

	var cmd = &cobra.Command
		Use:     "init",
		Short:   "Bootstraps cloud component. Checks and install (if required) the pre-requisites.",
		Long:    cloudBetaInitLongDescription,
		Example: cloudBetaInitExample,
		RunE: func(cmd *cobra.Command, args []string) error 
			checkFlags := func(f *pflag.Flag) 
				util.AddToolVals(f, flagVals)
			
			cmd.Flags().VisitAll(checkFlags)
			err := AddInitBeta2ToolsList(tools, BetaInit)
			if err != nil 
				return err
			
			return ExecuteInitBeta(tools)
		,
	
...
	return cmd

func AddInitBeta2ToolsList(toolList map[string]types.ToolsInstaller, initBetaOpts *types.InitBetaOptions) error 

1、 checkFlags：检查命令行传入的参数

2、GetLatestVersion return the latest non-prerelease, non-draft version of kubeedge in release

若没有获取到最新版本，默认使用1.10.0版本。其次根据initBetaOpts配置一些参数设置

toolList["helm"] = &helm.KubeCloudHelmInstTool。

3、ExecuteInitBeta：使用helm安装cloudcore

func (cu *KubeCloudHelmInstTool) InstallTools() error 

两种安装方式：RunHelmInstall、RunHelmManifest

// InstallTools downloads KubeEdge for the specified version
// and makes the required configuration changes and initiates cloudcore.
func (cu *KubeCloudHelmInstTool) InstallTools() error 
	cu.SetOSInterface(util.GetOSInterface())
	cu.SetKubeEdgeVersion(cu.ToolVersion)

	...

	switch cu.Action 
	case types.HelmInstallAction:
		if err := cu.RunHelmInstall(baseHelmRoot); err != nil 
			return err
		
	case types.HelmManifestAction:
		if err := cu.RunHelmManifest(baseHelmRoot); err != nil 
			return err
		
	default:
		fmt.Println("Not support this action")
	

	return nil

1、RunHelmInstall renders the Charts with the given values, then installs the Charts to the cluster.
--force强制安装无论cloudcore是否存在。
// --force would not care about whether the cloud components exist or not
--external helm root 也会强制安装
// Also, if gives a external helm root, no need to check and verify. Because it is always not a cloudcore.

func (cu *KubeCloudHelmInstTool) RunHelmInstall(baseHelmRoot string) error 
	//检查cloudcore是否已经运行以及k8s组件是否安装。
	...
	// prepare to render
	if err := cu.beforeRenderer(baseHelmRoot); err != nil 
		return err
	

	// build a renderer instance with the given values and flagvals
	renderer, err := cu.buildRenderer(baseHelmRoot)
	if err != nil 
		return fmt.Errorf("cannot build renderer: %s", err.Error())
	

	release, err := cu.runHelmInstall(renderer)
	if err != nil 
		return err
	
    ......

	return nil

func (cu *KubeCloudHelmInstTool) beforeRenderer(baseHelmRoot string) error //combine the flag values
if cu.AdvertiseAddress != "" 
   for index, addr := range strings.Split(cu.AdvertiseAddress, ",") 
      cu.Sets = append(cu.Sets, fmt.Sprintf("%s[%d]=%s", "cloudCore.modules.cloudHub.advertiseAddress", index, addr))
   

// combineProfVals combines the values of the given manifests and flags into a map.
func (cu *KubeCloudHelmInstTool) combineProfVals() (map[string]interface, error) 把values.xml 以及 --set cloudCore.modules.dynamicController.enable=true命令行指定的flag存入map profileValue

    profileValue, err := loadValues(cu.ExternalHelmRoot, profilekey, cu.existsProfile)
	if err != nil 
		return nil, fmt.Errorf("cannot load profile yaml:%s", err.Error())
	

	if err := yaml.Unmarshal([]byte(profileValue), &profileValsMap); err != nil 
		return nil, fmt.Errorf("failed to unmarshal values: %v", err)
	
	// User specified a value via --set
	for _, value := range cu.Sets 
		if err := strvals.ParseInto(value, profileValsMap); err != nil 
			return nil, fmt.Errorf("failed parsing --set data:%s", err.Error())
		
	

// buildRenderer returns a renderer instance 返回一个renderer实例化对象。
// runHelmInstall starts cloudcore deployment with the given flags

func (cu *KubeCloudHelmInstTool) runHelmInstall(r *Renderer) (*release.Release, error) 
	....//主要是参数配置
	if performInstall 
		helmInstall := action.NewInstall(cfg)
		helmInstall.DryRun = cu.DryRun
		helmInstall.Namespace = cu.Namespace
		// --force would not wait.
		if !cu.Force 
			helmInstall.Wait = DefaultHelmWait
			helmInstall.Timeout = DefaultHelmTimeout
		
		helmInstall.CreateNamespace = DefaultHelmCreateNs
		helmInstall.ReleaseName = r.componentName
        //helmInstall通过r.chart, r.profileValsMap参数配置进行安装
		rel, err := helmInstall.Run(r.chart, r.profileValsMap)
		if err != nil 
			return nil, err
		
		return rel, nil
	

	// try to update a version
	...
	return rel, nil

keadm beta init 安装cloudcore过程中遇到的一个问题：

keadm beta init --advertise-address=124.70.221.xxx --force --profile version=v1.10.0

kubectl get pod -n kubeedge:   cloudcore的状态是Pending

[root@huawei-node2 kubeedge-scripts]# kubectl get pod -n kubeedge
NAME                         READY   STATUS    RESTARTS   AGE
cloudcore-7d5655d6f4-qjfmn   0/1     Pending   0          4s

kubectl describe pod -n kubeedge :

"1 node(s) had taint node-role.kubernetes.io/master: , that the pod didn't tolerate."

 Warning  FailedScheduling  30s (x3 over 2m58s)  default-scheduler  
0/1 nodes are available: 1 node(s) had taint node-role.kubernetes.io/master: ,
 that the pod didn't tolerate.

问题原因是：

使用kubeadm初始化的集群，出于安全考虑Pod不会被调度到Master Node上，也就是说Master Node不参与工作负载。

这里搭建的是测试环境可以使用下面的命令使Master Node参与工作负载：
k8s是master节点的hostname
允许master节点部署pod，使用命令如下:

kubectl taint nodes --all node-role.kubernetes.io/master-

接下来，在keadm reset --force 之后重新keadm beta init 便可使cloudcore的状态正常为Running。