[ k8s-operator 系列 ] 自动生成 webhook 证书并在过期时自动刷新

Posted 2021-09-14 无风

k8s webhook 要求必须使用 HTTPS，Operator webhook 的证书配置是个麻烦事。

众所周知 cert-manager 可以帮我们生成证书，但证书生成后还需要修改 ValidatingWebhook 和 MutingWebhook。

下面要介绍的是另一种方式：github.com/open-policy-agent/cert-controller，最初是在 open-policy-agent/gatekeeper 中看到这种用法，感觉非常 Geek。

Operator 内置：

• 证书自动生成
• 过期刷新
• webhook 配置中的 caBundle 自动注入

不依赖外部组件，真正做到一键部署 Operator，非常方便分发 Operator。

使用

先启动 rotator：

setupFinished := make(chan struct{})
if !disableCertRotation {
  setupLog.Info("setting up cert rotation")
  err := rotator.AddRotator(mgr, &rotator.CertRotator{
    SecretKey: types.NamespacedName{
      Namespace: k8s.GetOperatorNamespace(),
      Name:      secretName,
    },
    CertDir:        certDir,
    CAName:         caName,
    CAOrganization: caOrganization,
    DNSName:        dnsName,
    IsReady:        setupFinished,
    Webhooks:       webhooks,
  })
  if err != nil {
    setupLog.Error(err, "unable to set up cert rotation")
    os.Exit(1)
  }
} else {
  close(setupFinished)
}

if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
  setupLog.Error(err, "problem running manager")
  os.Exit(1)
}

setupFinished 之后再启动 Reconciler 和 Webhook server：

go func() {
  <-setupFinished

  err := (&controllers.EtcdReconciler{
    Client: mgr.GetClient(),
    Log:    zaplog.Sugar().Named("controllers").Named("Etcd"),
    Scheme: mgr.GetScheme(),
  }).SetupWithManager(mgr)
  if err != nil {
    setupLog.Error(err, "unable to create controller", "controller", "Etcd")
    os.Exit(1)
  }

  if certDir != "" {
    err = (&dbv1.Etcd{}).SetupWebhookWithManager(mgr)
    if err != nil {
      setupLog.Error(err, "unable to SetupWebhookWithManager")
      os.Exit(1)
    }
  }
}()

controllerManager 有个特性，如果已经启动，则后面 Add 的 Runnable 直接启动：

// Add sets dependencies on i, and adds it to the list of Runnables to start.
func (cm *controllerManager) Add(r Runnable) error {
    // ...

    if shouldStart {
        // If already started, start the controller
        cm.startRunnable(r)
    }

    return nil
}

完整例子：win5do/etcd-operator (github.com)

原理

Webhook 使用的证书不要求一定是 k8s 集群 CA 根证书颁发的证书，所以我们可以使用自签证书。

生成自签 CA 根证书：

// CreateCACert creates the self-signed CA cert and private key that will
// be used to sign the server certificate
func (cr *CertRotator) CreateCACert(begin, end time.Time) (*KeyPairArtifacts, error) {
    // ...
    key, err := rsa.GenerateKey(rand.Reader, 2048)
    if err != nil {
        return nil, errors.Wrap(err, "generating key")
    }
    der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
    if err != nil {
        return nil, errors.Wrap(err, "creating certificate")
    }
    certPEM, keyPEM, err := pemEncode(der, key)
    if err != nil {
        return nil, errors.Wrap(err, "encoding PEM")
    }
    cert, err := x509.ParseCertificate(der)
    if err != nil {
        return nil, errors.Wrap(err, "parsing certificate")
    }

    return &KeyPairArtifacts{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil
}

根据 service-name.namespace.svc 生成服务器 tls.key 和 tls.crt：

// CreateCertPEM takes the results of CreateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func (cr *CertRotator) CreateCertPEM(ca *KeyPairArtifacts, begin, end time.Time) ([]byte, []byte, error) {
    // ...
    key, err := rsa.GenerateKey(rand.Reader, 2048)
    if err != nil {
        return nil, nil, errors.Wrap(err, "generating key")
    }
    der, err := x509.CreateCertificate(rand.Reader, templ, ca.Cert, key.Public(), ca.Key)
    if err != nil {
        return nil, nil, errors.Wrap(err, "creating certificate")
    }
    certPEM, keyPEM, err := pemEncode(der, key)
    if err != nil {
        return nil, nil, errors.Wrap(err, "encoding PEM")
    }
    return certPEM, keyPEM, nil
}

写入到 secret 中， 并等待 mount：

// ensureCertsMounted ensure the cert files exist.
func (cr *CertRotator) ensureCertsMounted() {
    checkFn := func() (bool, error) {
        certFile := cr.CertDir + "/" + certName
        _, err := os.Stat(certFile)
        if err == nil {
            return true, nil
        }
        return false, nil
    }
    if err := wait.ExponentialBackoff(wait.Backoff{
        Duration: 1 * time.Second,
        Factor:   2,
        Jitter:   1,
        Steps:    10,
    }, checkFn); err != nil {
        crLog.Error(err, "max retries for checking certs existence")
        close(cr.certsNotMounted)
        return
    }
    crLog.Info(fmt.Sprintf("certs are ready in %s", cr.CertDir))
    close(cr.certsMounted)
}

启动一个 Reconceile 监听 secret，如果有更新则将 caBundle 注入到 webhook config 中：

func injectCertToWebhook(wh *unstructured.Unstructured, certPem []byte) error {
    webhooks, found, err := unstructured.NestedSlice(wh.Object, "webhooks")
    if err != nil {
        return err
    }
    if !found {
        return errors.New("`webhooks` field not found in ValidatingWebhookConfiguration")
    }
    for i, h := range webhooks {
        hook, ok := h.(map[string]interface{})
        if !ok {
            return errors.Errorf("webhook %d is not well-formed", i)
        }
        if err := unstructured.SetNestedField(hook, base64.StdEncoding.EncodeToString(certPem), "clientConfig", "caBundle"); err != nil {
            return err
        }
        webhooks[i] = hook
    }
    if err := unstructured.SetNestedSlice(wh.Object, webhooks, "webhooks"); err != nil {
        return err
    }
    return nil
}