Skip to content

Commit

Permalink
Avoid csr name clash (#89)
Browse files Browse the repository at this point in the history 
* Remove v1beta1 related code, this API has been removed from k8s

* Add a configurable timeout to the csr container. Once expired, it will crash-loop, making the pod re-issue a CSR.

* Add label to help operator filter tigera csr's in watches and reconcile loops.
  • Loading branch information
@rene-dekker
rene-dekker authored Dec 14, 2023
1 parent b5eba73 commit c0fec19
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 248 deletions.
3 changes: 2 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ Makefile.common.$(MAKE_BRANCH):
rm -f Makefile.common.*
curl --fail $(MAKE_REPO)/Makefile.common -o "$@"

GOFLAGS = -buildvcs=false
include Makefile.common

# Build a static binary with boring crypto support.
Expand All @@ -60,7 +61,7 @@ define build_static_cgo_boring_binary
$(GO_BUILD_IMAGE):$(GO_BUILD_VER) \
sh -c '$(GIT_CONFIG_SSH) \
GOEXPERIMENT=boringcrypto go build -o $(2) \
-tags fipsstrict,osusergo,netgo$(if $(BUILD_TAGS),$(comma)$(BUILD_TAGS)) -v -buildvcs=false \
-tags fipsstrict,osusergo,netgo$(if $(BUILD_TAGS),$(comma)$(BUILD_TAGS)) -v \
-ldflags "$(LDFLAGS) -linkmode external -extldflags -static -s -w" \
$(1) \
&& strings $(2) | grep '_Cfunc__goboringcrypto_' 1> /dev/null'
Expand Down
49 changes: 34 additions & 15 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package main
import (
"context"
"flag"
"os"

log "github.com/sirupsen/logrus"

Expand All @@ -31,24 +32,42 @@ func main() {
log.SetReportCaller(true)
// Initiate (and validate) env variables
config := cfg.GetConfigOrDie()
ctx := context.Background()
ctx, cancel := context.WithTimeout(context.TODO(), config.TimeoutDuration)
defer cancel()

// Initiate REST restClient
restClient, err := k8s.NewRestClient()
if err != nil {
log.WithError(err).Fatalf("Unable to create a kubernetes rest restClient")
}
// Make a routine that runs and signals the channel. This allows us to end the task if we run out of time.
channelWithTimeout := make(chan bool, 1)
go func() {
// Initiate REST restClient
restClient, err := k8s.NewRestClient()
if err != nil {
log.WithError(err).Fatalf("Unable to create a kubernetes rest restClient")
}

csr, err := tls.CreateX509CSR(config)
if err != nil {
log.WithError(err).Fatalf("Unable to create x509 certificate request")
}
csr, err := tls.CreateX509CSR(config)
if err != nil {
log.WithError(err).Fatalf("Unable to create x509 certificate request")
}

if err := k8s.SubmitCSR(ctx, config, restClient, csr); err != nil {
log.WithError(err).Fatalf("Unable to submit a CSR")
}
if err := k8s.SubmitCSR(ctx, config, restClient, csr); err != nil {
log.WithError(err).Fatalf("Unable to submit a CSR")
}

if err := k8s.WatchCSR(ctx, restClient, config, csr); err != nil {
log.WithError(err).Fatalf("Unable to watch CSR")
}
// Signal the channel that we completed the task within the designated deadline.
channelWithTimeout <- true
}()

if err := k8s.WatchCSR(ctx, restClient, config, csr); err != nil {
log.WithError(err).Fatalf("Unable to watch CSR")
// Wait for the work to finish. If it takes too we crash-loop to improve the odds of the pod eventually getting up and running.
select {
case <-channelWithTimeout:
log.Info("successfully obtained a certificate")
channelWithTimeout <- true
case <-ctx.Done():
// If we reach here, it means that we were not able to obtain a certificate within config.TimeoutDuration.
log.Fatal("timeout expired, exiting program with exit code 1")
os.Exit(1)
}
}
21 changes: 20 additions & 1 deletion pkg/cfg/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"fmt"
"os"
"strings"
"time"

log "github.com/sirupsen/logrus"
)
Expand All @@ -40,6 +41,7 @@ type Config struct {
PrivateKeyAlgorithm string
RegisterApiserver bool
AppName string
TimeoutDuration time.Duration
}

// GetEnvOrDie convenience method for initializing env.
Expand Down Expand Up @@ -70,9 +72,25 @@ func GetConfigOrDie() *Config {
}
caCertName = GetEnvOrDie("CA_CERT_NAME")
}
secretName := os.Getenv("SECRET_NAME")
var csrName string
if secretName == "" {
csrName = fmt.Sprintf("%s:%s", GetEnvOrDie("POD_NAMESPACE"), GetEnvOrDie("POD_NAME"))
} else {
csrName = fmt.Sprintf("%s:%s", secretName, GetEnvOrDie("POD_NAME"))
}
timeoutDuration := 90 * time.Second
timeoutEnv := os.Getenv("TIMEOUT_DURATION")
if timeoutEnv != "" {
var err error
timeoutDuration, err = time.ParseDuration(timeoutEnv)
if err != nil {
log.Fatalf("unable to convert TIMEOUT env to an integer: %v", err)
}

}
return &Config{
CSRName: fmt.Sprintf("%s:%s", GetEnvOrDie("POD_NAMESPACE"), GetEnvOrDie("POD_NAME")),
CSRName: csrName,
SignatureAlgorithm: os.Getenv("SIGNATURE_ALGORITHM"),
Signer: GetEnvOrDie("SIGNER"),
CommonName: GetEnvOrDie("COMMON_NAME"),
Expand All @@ -86,5 +104,6 @@ func GetConfigOrDie() *Config {
AppName: GetEnvOrDie("APP_NAME"),
PrivateKeyAlgorithm: os.Getenv("KEY_ALGORITHM"),
DNSNames: dnsNames,
TimeoutDuration: timeoutDuration,
}
}
145 changes: 5 additions & 140 deletions pkg/k8s/certificate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,61 +19,28 @@ import (
"fmt"
"os"
"path"
"strconv"
"strings"

log "github.com/sirupsen/logrus"
"github.com/tigera/key-cert-provisioner/pkg/cfg"
"github.com/tigera/key-cert-provisioner/pkg/tls"

certV1 "k8s.io/api/certificates/v1"
certV1beta1 "k8s.io/api/certificates/v1beta1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)

type versionInfo struct {
Major int
Minor int
}

// WatchCSR Watches the CSR resource for updates and writes results to the certificate location (which should be mounted as an emptyDir)
func WatchCSR(ctx context.Context, restClient *RestClient, cfg *cfg.Config, x509CSR *tls.X509CSR) error {
version, err := GetKubernetesVersion(restClient.Clientset)
if err != nil {
return err
}

watcher, err := createCSRWatcher(ctx, restClient, version)
watcher, err := restClient.Clientset.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{})
if err != nil {
return fmt.Errorf("unable to watch certificate requests: %w", err)
}
log.Infof("watching CSR until it has been signed and approved: %v", cfg.CSRName)
return watchCSRBasedOnKubernetesVersion(watcher, cfg, x509CSR, version)
return watchCSR(&watcher, cfg, x509CSR)
}

func createCSRWatcher(ctx context.Context, restClient *RestClient, version *versionInfo) (*watch.Interface, error) {
var watcher watch.Interface
var err error
if version.Major > 1 || version.Minor >= 19 {
watcher, err = restClient.Clientset.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{})
} else {
watcher, err = restClient.Clientset.CertificatesV1beta1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{})
}
return &watcher, err
}

func watchCSRBasedOnKubernetesVersion(watcher *watch.Interface, cfg *cfg.Config, x509CSR *tls.X509CSR, version *versionInfo) error {
if version.Major > 1 || version.Minor >= 19 {
return watchCSRUsingCertV1(watcher, cfg, x509CSR)
}
return watchCSRUsingCertV1beta1(watcher, cfg, x509CSR)
}

func watchCSRUsingCertV1(watcher *watch.Interface, cfg *cfg.Config, x509CSR *tls.X509CSR) error {
func watchCSR(watcher *watch.Interface, cfg *cfg.Config, x509CSR *tls.X509CSR) error {
for event := range (*watcher).ResultChan() {
chcsr, ok := event.Object.(*certV1.CertificateSigningRequest)
if !ok {
Expand Down Expand Up @@ -101,35 +68,6 @@ func watchCSRUsingCertV1(watcher *watch.Interface, cfg *cfg.Config, x509CSR *tls
return nil
}

func watchCSRUsingCertV1beta1(watcher *watch.Interface, cfg *cfg.Config, x509CSR *tls.X509CSR) error {
for event := range (*watcher).ResultChan() {
chcsr, ok := event.Object.(*certV1beta1.CertificateSigningRequest)
if !ok {
return fmt.Errorf("unexpected type in CertificateSigningRequest channel: %o", event.Object)
}
if chcsr.Name == cfg.CSRName && chcsr.Status.Conditions != nil && len(chcsr.Status.Certificate) > 0 {
approved := false
for _, c := range chcsr.Status.Conditions {
//status unset should be treated as true for backwards compatibility.
if c.Type == certV1beta1.CertificateApproved && (c.Status == v1.ConditionTrue || c.Status == "") {
approved = true
break
}
if c.Type == certV1beta1.CertificateDenied && c.Status == v1.ConditionTrue {
return fmt.Errorf("CSR was denied for this pod. CSR name: %s", cfg.CSRName)
}
if c.Type == certV1beta1.CertificateFailed && c.Status == v1.ConditionTrue {
return fmt.Errorf("CSR failed for this pod. CSR name: %s", cfg.CSRName)
}
}
if approved {
return WriteCertificateToFile(cfg, chcsr.Status.Certificate, x509CSR)
}
}
}
return nil
}

// WriteCertificateToFile writes TLS key, cert and cacert to the mount location specified in the config parameter.
func WriteCertificateToFile(cfg *cfg.Config, cert []byte, x509CSR *tls.X509CSR) error {
log.Infof("the CSR has been signed and approved, writing to certificate location: %v", cfg.EmptyDirLocation)
Expand Down Expand Up @@ -158,29 +96,15 @@ func WriteCertificateToFile(cfg *cfg.Config, cert []byte, x509CSR *tls.X509CSR)

// SubmitCSR Submits a CSR in order to obtain a signed certificate for this pod.
func SubmitCSR(ctx context.Context, config *cfg.Config, restClient *RestClient, x509CSR *tls.X509CSR) error {
version, err := GetKubernetesVersion(restClient.Clientset)
if err != nil {
return err
}

return submitCSRBasedOnKubernetesVersion(ctx, config, restClient, x509CSR, version)
}

func submitCSRBasedOnKubernetesVersion(ctx context.Context, config *cfg.Config, restClient *RestClient, x509CSR *tls.X509CSR, version *versionInfo) error {
if version.Major > 1 || version.Minor >= 19 {
return submitCSRUsingCertV1(ctx, config, restClient, x509CSR)
}
return submitCSRUsingCertV1beta1(ctx, config, restClient, x509CSR)
}

func submitCSRUsingCertV1(ctx context.Context, config *cfg.Config, restClient *RestClient, x509CSR *tls.X509CSR) error {
cli := restClient.Clientset.CertificatesV1().CertificateSigningRequests()
csr := &certV1.CertificateSigningRequest{
TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest", APIVersion: "certificates.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
Name: config.CSRName,
Labels: map[string]string{
"k8s-app": config.AppName,
// Add a label so that our controller can filter on CSRs issued by tigera.
"operator.tigera.io/csr": config.AppName,
}},
Spec: certV1.CertificateSigningRequestSpec{
Request: x509CSR.CSR,
Expand Down Expand Up @@ -208,62 +132,3 @@ func submitCSRUsingCertV1(ctx context.Context, config *cfg.Config, restClient *R
log.Infof("created CSR: %v", created)
return nil
}

func submitCSRUsingCertV1beta1(ctx context.Context, config *cfg.Config, restClient *RestClient, x509CSR *tls.X509CSR) error {
cli := restClient.Clientset.CertificatesV1beta1().CertificateSigningRequests()
csr := &certV1beta1.CertificateSigningRequest{
TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest", APIVersion: "certificates.k8s.io/v1beta1"},
ObjectMeta: metav1.ObjectMeta{
Name: config.CSRName,
Labels: map[string]string{
"k8s-app": config.AppName,
}},
Spec: certV1beta1.CertificateSigningRequestSpec{
Request: x509CSR.CSR,
SignerName: &config.Signer,
Usages: []certV1beta1.KeyUsage{certV1beta1.UsageServerAuth, certV1beta1.UsageClientAuth, certV1beta1.UsageDigitalSignature, certV1beta1.UsageKeyAgreement},
},
}

created, err := cli.Create(ctx, csr, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
// If this is the case, it means this pod crashed previously. We need to delete the CSR and re-submit a new CSR,
// otherwise we end up with a private key that does not match the issued cert.
if err = cli.Delete(ctx, config.CSRName, metav1.DeleteOptions{}); err != nil {
return err
}
if created, err = cli.Create(ctx, csr, metav1.CreateOptions{}); err != nil {
return err
}
} else {
return fmt.Errorf("crashed while trying to create Kubernetes certificate signing request: %w", err)
}
}

log.Infof("created CSR: %v", created)
return nil
}

func GetKubernetesVersion(clientset kubernetes.Interface) (*versionInfo, error) {
v, err := clientset.Discovery().ServerVersion()
if err != nil {
return nil, fmt.Errorf("failed to check k8s version: %v", err)
}

major, err := strconv.Atoi(v.Major)
if err != nil {
return nil, fmt.Errorf("failed to parse k8s major version: %s", v.Major)
}

// filter out a proceeding '+' from the minor version since openshift includes that.
minor, err := strconv.Atoi(strings.TrimSuffix(v.Minor, "+"))
if err != nil {
return nil, fmt.Errorf("failed to parse k8s minor version: %s", v.Minor)
}

return &versionInfo{
Major: major,
Minor: minor,
}, nil
}
Loading

0 comments on commit c0fec19

Please sign in to comment.