Merge pull request #2025 from flant/kubernetes-apiextensions-version

feat: create CRDs as apiextensions.k8s.io/v1

go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	entgo.io/ent v0.8.0
 	github.com/AppsFlyer/go-sundheit v0.4.0
+	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/beevik/etree v1.1.0
 	github.com/coreos/go-oidc/v3 v3.0.0

go.sum

@@ -52,6 +52,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=

scripts/manifests/crds/authcodes.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: authcodes.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: AuthCodeList
     plural: authcodes
     singular: authcode
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/authrequests.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: authrequests.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: AuthRequestList
     plural: authrequests
     singular: authrequest
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/connectors.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: connectors.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: ConnectorList
     plural: connectors
     singular: connector
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/devicerequests.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: devicerequests.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: DeviceRequestList
     plural: devicerequests
     singular: devicerequest
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/devicetokens.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: devicetokens.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: DeviceTokenList
     plural: devicetokens
     singular: devicetoken
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/oauth2clients.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: oauth2clients.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: OAuth2ClientList
     plural: oauth2clients
     singular: oauth2client
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/offlinesessionses.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: offlinesessionses.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: OfflineSessionsList
     plural: offlinesessionses
     singular: offlinesessions
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/passwords.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: passwords.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: PasswordList
     plural: passwords
     singular: password
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/refreshtokens.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: refreshtokens.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: RefreshTokenList
     plural: refreshtokens
     singular: refreshtoken
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

scripts/manifests/crds/signingkeies.yaml

@@ -1,4 +1,4 @@
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: signingkeies.dex.coreos.com
@@ -9,4 +9,12 @@ spec:
     listKind: SigningKeyList
     plural: signingkeies
     singular: signingkey
-  version: v1
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true

storage/kubernetes/client.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver"
 	"github.com/ghodss/yaml"
 	"golang.org/x/net/http2"
 
@@ -47,6 +48,10 @@ type client struct {
 	// API version of the oidc resources. For example "oidc.coreos.com". This is
 	// currently not configurable, but could be in the future.
 	apiVersion string
+	// API version of the custom resource definitions.
+	// Different Kubernetes version requires to create CRD in certain API. It will be discovered automatically on
+	// storage opening.
+	crdAPIVersion string
 
 	// This is called once the client's Close method is called to signal goroutines,
 	// such as the one creating third party resources, to stop.
@@ -195,6 +200,37 @@ func (cli *client) postResource(apiVersion, namespace, resource string, v interf
 	return checkHTTPErr(resp, http.StatusCreated)
 }
 
+func (cli *client) detectKubernetesVersion() error {
+	var version struct{ GitVersion string }
+
+	url := cli.baseURL + "/version"
+	resp, err := cli.client.Get(url)
+	if err != nil {
+		return err
+	}
+
+	defer closeResp(resp)
+	if err := checkHTTPErr(resp, http.StatusOK); err != nil {
+		return err
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&version); err != nil {
+		return err
+	}
+
+	clusterVersion, err := semver.NewVersion(version.GitVersion)
+	if err != nil {
+		cli.logger.Warnf("cannot detect Kubernetes version (%s): %v", clusterVersion, err)
+		return nil
+	}
+
+	if clusterVersion.LessThan(semver.MustParse("v1.16.0")) {
+		cli.crdAPIVersion = legacyCRDAPIVersion
+	}
+
+	return nil
+}
+
 func (cli *client) delete(resource, name string) error {
 	url := cli.urlFor(cli.apiVersion, cli.namespace, resource, name)
 	req, err := http.NewRequest("DELETE", url, nil)
@@ -355,6 +391,7 @@ func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, l
 		hash:          func() hash.Hash { return fnv.New64() },
 		namespace:     namespace,
 		apiVersion:    apiVersion,
+		crdAPIVersion: crdAPIVersion,
 		logger:        logger,
 	}, nil
 }

storage/kubernetes/k8sapi/crd_extensions.go

@@ -26,6 +26,15 @@ type CustomResourceDefinitionSpec struct {
 
 	// Scope indicates whether this resource is cluster or namespace scoped.  Default is namespaced
 	Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"`
+	// versions is the list of all API versions of the defined custom resource.
+	// Version names are used to compute the order in which served versions are listed in API discovery.
+	// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+	// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+	// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+	// by GA > beta > alpha (where GA is a version with no suffix such as beta or alpha), and then by comparing
+	// major version, then minor version. An example sorted list of versions:
+	// v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+	Versions []CustomResourceDefinitionVersion `json:"versions" protobuf:"bytes,7,rep,name=versions"`
 }
 
 // CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@@ -139,3 +148,29 @@ type CustomResourceDefinitionList struct {
 	// Items individual CustomResourceDefinitions
 	Items []CustomResourceDefinition `json:"items" protobuf:"bytes,2,rep,name=items"`
 }
+
+type CustomResourceDefinitionVersion struct {
+	// name is the version name, e.g. “v1”, “v2beta1”, etc.
+	// The custom resources are served under this version at `/apis/<group>/<version>/...` if `served` is true.
+	Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
+	// served is a flag enabling/disabling this version from being served via REST APIs
+	Served bool `json:"served" protobuf:"varint,2,opt,name=served"`
+	// storage indicates this version should be used when persisting custom resources to storage.
+	// There must be exactly one version with storage=true.
+	Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"`
+	// schema describes the schema used for validation, pruning, and defaulting of this version of the custom resource.
+	// +optional
+	Schema *CustomResourceValidation `json:"schema,omitempty" protobuf:"bytes,4,opt,name=schema"`
+}
+
+// CustomResourceValidation is a list of validation methods for CustomResources.
+type CustomResourceValidation struct {
+	// OpenAPIV3Schema is the OpenAPI v3 schema to be validated against.
+	OpenAPIV3Schema *JSONSchemaProps `json:"openAPIV3Schema,omitempty" protobuf:"bytes,1,opt,name=openAPIV3Schema"`
+}
+
+// JSONSchemaProps is a JSON-Schema following Specification Draft 4 (http://json-schema.org/).
+type JSONSchemaProps struct {
+	Type                   string `json:"type,omitempty" protobuf:"bytes,5,opt,name=type"`
+	XPreserveUnknownFields *bool  `json:"x-kubernetes-preserve-unknown-fields,omitempty" protobuf:"bytes,38,opt,name=xKubernetesPreserveUnknownFields"`
+}

storage/kubernetes/storage.go

@@ -88,6 +88,10 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 		return nil, fmt.Errorf("create client: %v", err)
 	}
 
+	if err = cli.detectKubernetesVersion(); err != nil {
+		return nil, fmt.Errorf("cannot get kubernetes version: %v", err)
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 
 	logger.Info("creating custom Kubernetes resources")
@@ -100,7 +104,6 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 		// Try to synchronously create the custom resources once. This doesn't mean
 		// they'll immediately be available, but ensures that the client will actually try
 		// once.
-		logger.Errorf("failed creating custom resources: %v", err)
 		go func() {
 			for {
 				if cli.registerCustomResources() {
@@ -136,21 +139,25 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
 // Creating a custom resource does not mean that they'll be immediately available.
 func (cli *client) registerCustomResources() (ok bool) {
 	ok = true
-	length := len(customResourceDefinitions)
+
+	definitions := customResourceDefinitions(cli.crdAPIVersion)
+	length := len(definitions)
+
 	for i := 0; i < length; i++ {
 		var err error
 		var resourceName string
 
-		r := customResourceDefinitions[i]
+		r := definitions[i]
 		var i interface{}
-		cli.logger.Infof("checking if custom resource %s has been created already...", r.ObjectMeta.Name)
+		cli.logger.Infof("checking if custom resource %s has already been created...", r.ObjectMeta.Name)
 		if err := cli.list(r.Spec.Names.Plural, &i); err == nil {
 			cli.logger.Infof("The custom resource %s already available, skipping create", r.ObjectMeta.Name)
 			continue
 		} else {
 			cli.logger.Infof("failed to list custom resource %s, attempting to create: %v", r.ObjectMeta.Name, err)
 		}
-		err = cli.postResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", r)
+
+		err = cli.postResource(cli.crdAPIVersion, "", "customresourcedefinitions", r)
 		resourceName = r.ObjectMeta.Name
 
 		if err != nil {
@@ -177,7 +184,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
 
-	for _, crd := range customResourceDefinitions {
+	for _, crd := range customResourceDefinitions(cli.crdAPIVersion) {
 		for {
 			err := cli.isCRDReady(crd.Name)
 			if err == nil {
@@ -199,7 +206,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
 // isCRDReady determines if a CRD is ready by inspecting its conditions.
 func (cli *client) isCRDReady(name string) error {
 	var r k8sapi.CustomResourceDefinition
-	err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r)
+	err := cli.getResource(cli.crdAPIVersion, "", "customresourcedefinitions", name, &r)
 	if err != nil {
 		return fmt.Errorf("get crd %s: %v", name, err)
 	}

storage/kubernetes/types.go

@@ -10,16 +10,49 @@ import (
 	"github.com/dexidp/dex/storage/kubernetes/k8sapi"
 )
 
-var crdMeta = k8sapi.TypeMeta{
-	APIVersion: "apiextensions.k8s.io/v1beta1",
-	Kind:       "CustomResourceDefinition",
-}
+const (
+	apiGroup = "dex.coreos.com"
 
-const apiGroup = "dex.coreos.com"
+	legacyCRDAPIVersion = "apiextensions.k8s.io/v1beta1"
+	crdAPIVersion       = "apiextensions.k8s.io/v1"
+)
 
 // The set of custom resource definitions required by the storage. These are managed by
 // the storage so it can migrate itself by creating new resources.
-var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
+func customResourceDefinitions(apiVersion string) []k8sapi.CustomResourceDefinition {
+	crdMeta := k8sapi.TypeMeta{
+		APIVersion: apiVersion,
+		Kind:       "CustomResourceDefinition",
+	}
+
+	var version string
+	var scope k8sapi.ResourceScope
+	var versions []k8sapi.CustomResourceDefinitionVersion
+
+	switch apiVersion {
+	case crdAPIVersion:
+		preserveUnknownFields := true
+		versions = []k8sapi.CustomResourceDefinitionVersion{
+			{
+				Name:    "v1",
+				Served:  true,
+				Storage: true,
+				Schema: &k8sapi.CustomResourceValidation{
+					OpenAPIV3Schema: &k8sapi.JSONSchemaProps{
+						Type:                   "object",
+						XPreserveUnknownFields: &preserveUnknownFields,
+					},
+				},
+			},
+		}
+		scope = k8sapi.NamespaceScoped
+	case legacyCRDAPIVersion:
+		version = "v1"
+	default:
+		panic("unknown apiVersion " + apiVersion)
+	}
+
+	return []k8sapi.CustomResourceDefinition{
 		{
 			ObjectMeta: k8sapi.ObjectMeta{
 				Name: "authcodes.dex.coreos.com",
@@ -27,7 +60,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "authcodes",
 					Singular: "authcode",
@@ -42,7 +77,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "authrequests",
 					Singular: "authrequest",
@@ -57,7 +94,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "oauth2clients",
 					Singular: "oauth2client",
@@ -72,7 +111,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					// `signingkeies` is an artifact from the old TPR pluralization.
 					// Users don't directly interact with this value, hence leaving it
@@ -90,7 +131,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "refreshtokens",
 					Singular: "refreshtoken",
@@ -105,7 +148,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "passwords",
 					Singular: "password",
@@ -120,7 +165,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "offlinesessionses",
 					Singular: "offlinesessions",
@@ -135,7 +182,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "connectors",
 					Singular: "connector",
@@ -150,7 +199,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "devicerequests",
 					Singular: "devicerequest",
@@ -165,7 +216,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 			TypeMeta: crdMeta,
 			Spec: k8sapi.CustomResourceDefinitionSpec{
 				Group:    apiGroup,
-			Version: "v1",
+				Version:  version,
+				Versions: versions,
+				Scope:    scope,
 				Names: k8sapi.CustomResourceDefinitionNames{
 					Plural:   "devicetokens",
 					Singular: "devicetoken",
@@ -173,6 +226,7 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
 				},
 			},
 		},
+	}
 }
 
 // There will only ever be a single keys resource. Maintain this by setting a