feat: Create CRDs as apiextensions.k8s.io/v1

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
This commit is contained in:
m.nabokikh 2021-02-15 11:53:59 +04:00
parent baec4f79ce
commit 7a2472555a
16 changed files with 387 additions and 171 deletions

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.16
require (
entgo.io/ent v0.8.0
github.com/AppsFlyer/go-sundheit v0.4.0
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/sprig/v3 v3.2.2
github.com/beevik/etree v1.1.0
github.com/coreos/go-oidc/v3 v3.0.0

2
go.sum
View file

@ -51,6 +51,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: authcodes.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: AuthCodeList
plural: authcodes
singular: authcode
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: authrequests.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: AuthRequestList
plural: authrequests
singular: authrequest
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: connectors.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: ConnectorList
plural: connectors
singular: connector
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: devicerequests.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: DeviceRequestList
plural: devicerequests
singular: devicerequest
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: devicetokens.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: DeviceTokenList
plural: devicetokens
singular: devicetoken
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: oauth2clients.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: OAuth2ClientList
plural: oauth2clients
singular: oauth2client
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: offlinesessionses.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: OfflineSessionsList
plural: offlinesessionses
singular: offlinesessions
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: passwords.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: PasswordList
plural: passwords
singular: password
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: refreshtokens.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: RefreshTokenList
plural: refreshtokens
singular: refreshtoken
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -1,4 +1,4 @@
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: signingkeies.dex.coreos.com
@ -9,4 +9,12 @@ spec:
listKind: SigningKeyList
plural: signingkeies
singular: signingkey
version: v1
scope: Namespaced
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
x-kubernetes-preserve-unknown-fields: true

View file

@ -22,6 +22,7 @@ import (
"strings"
"time"
"github.com/Masterminds/semver"
"github.com/ghodss/yaml"
"golang.org/x/net/http2"
@ -47,6 +48,10 @@ type client struct {
// API version of the oidc resources. For example "oidc.coreos.com". This is
// currently not configurable, but could be in the future.
apiVersion string
// API version of the custom resource definitions.
// Different Kubernetes version requires to create CRD in certain API. It will be discovered automatically on
// storage opening.
crdAPIVersion string
// This is called once the client's Close method is called to signal goroutines,
// such as the one creating third party resources, to stop.
@ -195,6 +200,37 @@ func (cli *client) postResource(apiVersion, namespace, resource string, v interf
return checkHTTPErr(resp, http.StatusCreated)
}
func (cli *client) detectKubernetesVersion() error {
var version struct{ GitVersion string }
url := cli.baseURL + "/version"
resp, err := cli.client.Get(url)
if err != nil {
return err
}
defer closeResp(resp)
if err := checkHTTPErr(resp, http.StatusOK); err != nil {
return err
}
if err := json.NewDecoder(resp.Body).Decode(&version); err != nil {
return err
}
clusterVersion, err := semver.NewVersion(version.GitVersion)
if err != nil {
cli.logger.Warnf("cannot detect Kubernetes version (%s): %v", clusterVersion, err)
return nil
}
if clusterVersion.LessThan(semver.MustParse("v1.16.0")) {
cli.crdAPIVersion = legacyCRDAPIVersion
}
return nil
}
func (cli *client) delete(resource, name string) error {
url := cli.urlFor(cli.apiVersion, cli.namespace, resource, name)
req, err := http.NewRequest("DELETE", url, nil)
@ -355,6 +391,7 @@ func newClient(cluster k8sapi.Cluster, user k8sapi.AuthInfo, namespace string, l
hash: func() hash.Hash { return fnv.New64() },
namespace: namespace,
apiVersion: apiVersion,
crdAPIVersion: crdAPIVersion,
logger: logger,
}, nil
}

View file

@ -26,6 +26,15 @@ type CustomResourceDefinitionSpec struct {
// Scope indicates whether this resource is cluster or namespace scoped. Default is namespaced
Scope ResourceScope `json:"scope" protobuf:"bytes,4,opt,name=scope,casttype=ResourceScope"`
// versions is the list of all API versions of the defined custom resource.
// Version names are used to compute the order in which served versions are listed in API discovery.
// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
// by GA > beta > alpha (where GA is a version with no suffix such as beta or alpha), and then by comparing
// major version, then minor version. An example sorted list of versions:
// v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
Versions []CustomResourceDefinitionVersion `json:"versions" protobuf:"bytes,7,rep,name=versions"`
}
// CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@ -139,3 +148,29 @@ type CustomResourceDefinitionList struct {
// Items individual CustomResourceDefinitions
Items []CustomResourceDefinition `json:"items" protobuf:"bytes,2,rep,name=items"`
}
type CustomResourceDefinitionVersion struct {
// name is the version name, e.g. “v1”, “v2beta1”, etc.
// The custom resources are served under this version at `/apis/<group>/<version>/...` if `served` is true.
Name string `json:"name" protobuf:"bytes,1,opt,name=name"`
// served is a flag enabling/disabling this version from being served via REST APIs
Served bool `json:"served" protobuf:"varint,2,opt,name=served"`
// storage indicates this version should be used when persisting custom resources to storage.
// There must be exactly one version with storage=true.
Storage bool `json:"storage" protobuf:"varint,3,opt,name=storage"`
// schema describes the schema used for validation, pruning, and defaulting of this version of the custom resource.
// +optional
Schema *CustomResourceValidation `json:"schema,omitempty" protobuf:"bytes,4,opt,name=schema"`
}
// CustomResourceValidation is a list of validation methods for CustomResources.
type CustomResourceValidation struct {
// OpenAPIV3Schema is the OpenAPI v3 schema to be validated against.
OpenAPIV3Schema *JSONSchemaProps `json:"openAPIV3Schema,omitempty" protobuf:"bytes,1,opt,name=openAPIV3Schema"`
}
// JSONSchemaProps is a JSON-Schema following Specification Draft 4 (http://json-schema.org/).
type JSONSchemaProps struct {
Type string `json:"type,omitempty" protobuf:"bytes,5,opt,name=type"`
XPreserveUnknownFields *bool `json:"x-kubernetes-preserve-unknown-fields,omitempty" protobuf:"bytes,38,opt,name=xKubernetesPreserveUnknownFields"`
}

View file

@ -88,6 +88,10 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
return nil, fmt.Errorf("create client: %v", err)
}
if err = cli.detectKubernetesVersion(); err != nil {
return nil, fmt.Errorf("cannot get kubernetes version: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
logger.Info("creating custom Kubernetes resources")
@ -100,7 +104,6 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
// Try to synchronously create the custom resources once. This doesn't mean
// they'll immediately be available, but ensures that the client will actually try
// once.
logger.Errorf("failed creating custom resources: %v", err)
go func() {
for {
if cli.registerCustomResources() {
@ -136,21 +139,25 @@ func (c *Config) open(logger log.Logger, waitForResources bool) (*client, error)
// Creating a custom resource does not mean that they'll be immediately available.
func (cli *client) registerCustomResources() (ok bool) {
ok = true
length := len(customResourceDefinitions)
definitions := customResourceDefinitions(cli.crdAPIVersion)
length := len(definitions)
for i := 0; i < length; i++ {
var err error
var resourceName string
r := customResourceDefinitions[i]
r := definitions[i]
var i interface{}
cli.logger.Infof("checking if custom resource %s has been created already...", r.ObjectMeta.Name)
cli.logger.Infof("checking if custom resource %s has already been created...", r.ObjectMeta.Name)
if err := cli.list(r.Spec.Names.Plural, &i); err == nil {
cli.logger.Infof("The custom resource %s already available, skipping create", r.ObjectMeta.Name)
continue
} else {
cli.logger.Infof("failed to list custom resource %s, attempting to create: %v", r.ObjectMeta.Name, err)
}
err = cli.postResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", r)
err = cli.postResource(cli.crdAPIVersion, "", "customresourcedefinitions", r)
resourceName = r.ObjectMeta.Name
if err != nil {
@ -177,7 +184,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
for _, crd := range customResourceDefinitions {
for _, crd := range customResourceDefinitions(cli.crdAPIVersion) {
for {
err := cli.isCRDReady(crd.Name)
if err == nil {
@ -199,7 +206,7 @@ func (cli *client) waitForCRDs(ctx context.Context) error {
// isCRDReady determines if a CRD is ready by inspecting its conditions.
func (cli *client) isCRDReady(name string) error {
var r k8sapi.CustomResourceDefinition
err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r)
err := cli.getResource(cli.crdAPIVersion, "", "customresourcedefinitions", name, &r)
if err != nil {
return fmt.Errorf("get crd %s: %v", name, err)
}

View file

@ -10,16 +10,49 @@ import (
"github.com/dexidp/dex/storage/kubernetes/k8sapi"
)
var crdMeta = k8sapi.TypeMeta{
APIVersion: "apiextensions.k8s.io/v1beta1",
Kind: "CustomResourceDefinition",
}
const (
apiGroup = "dex.coreos.com"
const apiGroup = "dex.coreos.com"
legacyCRDAPIVersion = "apiextensions.k8s.io/v1beta1"
crdAPIVersion = "apiextensions.k8s.io/v1"
)
// The set of custom resource definitions required by the storage. These are managed by
// the storage so it can migrate itself by creating new resources.
var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
func customResourceDefinitions(apiVersion string) []k8sapi.CustomResourceDefinition {
crdMeta := k8sapi.TypeMeta{
APIVersion: apiVersion,
Kind: "CustomResourceDefinition",
}
var version string
var scope k8sapi.ResourceScope
var versions []k8sapi.CustomResourceDefinitionVersion
switch apiVersion {
case crdAPIVersion:
preserveUnknownFields := true
versions = []k8sapi.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: &k8sapi.CustomResourceValidation{
OpenAPIV3Schema: &k8sapi.JSONSchemaProps{
Type: "object",
XPreserveUnknownFields: &preserveUnknownFields,
},
},
},
}
scope = k8sapi.NamespaceScoped
case legacyCRDAPIVersion:
version = "v1"
default:
panic("unknown apiVersion " + apiVersion)
}
return []k8sapi.CustomResourceDefinition{
{
ObjectMeta: k8sapi.ObjectMeta{
Name: "authcodes.dex.coreos.com",
@ -27,7 +60,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "authcodes",
Singular: "authcode",
@ -42,7 +77,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "authrequests",
Singular: "authrequest",
@ -57,7 +94,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "oauth2clients",
Singular: "oauth2client",
@ -72,7 +111,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
// `signingkeies` is an artifact from the old TPR pluralization.
// Users don't directly interact with this value, hence leaving it
@ -90,7 +131,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "refreshtokens",
Singular: "refreshtoken",
@ -105,7 +148,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "passwords",
Singular: "password",
@ -120,7 +165,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "offlinesessionses",
Singular: "offlinesessions",
@ -135,7 +182,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "connectors",
Singular: "connector",
@ -150,7 +199,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "devicerequests",
Singular: "devicerequest",
@ -165,7 +216,9 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
TypeMeta: crdMeta,
Spec: k8sapi.CustomResourceDefinitionSpec{
Group: apiGroup,
Version: "v1",
Version: version,
Versions: versions,
Scope: scope,
Names: k8sapi.CustomResourceDefinitionNames{
Plural: "devicetokens",
Singular: "devicetoken",
@ -174,6 +227,7 @@ var customResourceDefinitions = []k8sapi.CustomResourceDefinition{
},
},
}
}
// There will only ever be a single keys resource. Maintain this by setting a
// common name.