From 3d2d92b31be5e2d20abcbe7729cf20dfe33e9ed0 Mon Sep 17 00:00:00 2001 From: Eric Chiang Date: Tue, 19 Sep 2017 15:31:58 -0700 Subject: [PATCH] *: run kubernetes tests in travis --- .travis.yml | 2 + Documentation/dev-integration-tests.md | 8 +++- scripts/test-k8s.sh | 56 ++++++++++++++++++++++++ storage/kubernetes/client.go | 6 ++- storage/kubernetes/storage.go | 59 +++++++++++++++++++++++--- 5 files changed, 122 insertions(+), 9 deletions(-) create mode 100755 scripts/test-k8s.sh diff --git a/.travis.yml b/.travis.yml index 3dc29262..f5554e59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ go: services: - postgresql + - docker env: - DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost" DEX_LDAP_TESTS=1 DEBIAN_FRONTEND=noninteractive @@ -21,6 +22,7 @@ install: script: - make testall + - ./scripts/test-k8s.sh notifications: email: false diff --git a/Documentation/dev-integration-tests.md b/Documentation/dev-integration-tests.md index 366b4ae3..a9ce1ad3 100644 --- a/Documentation/dev-integration-tests.md +++ b/Documentation/dev-integration-tests.md @@ -2,7 +2,7 @@ ## Kubernetes -Kubernetes tests will only run if the `DEX_KUBECONFIG` environment variable is set. +Kubernetes tests run against a Kubernetes API server, and are enabled by the `DEX_KUBECONFIG` environment variable: ``` $ export DEX_KUBECONFIG=~/.kube/config @@ -10,7 +10,11 @@ $ go test -v -i ./storage/kubernetes $ go test -v ./storage/kubernetes ``` -Because third party resources creation isn't synchronized it's expected that the tests fail the first time. Fear not, and just run them again. +These tests can be executed locally using docker by running the following script: + +``` +$ ./scripts/test-k8s.sh +``` ## Postgres diff --git a/scripts/test-k8s.sh b/scripts/test-k8s.sh new file mode 100755 index 00000000..86127be1 --- /dev/null +++ b/scripts/test-k8s.sh @@ -0,0 +1,56 @@ +#!/bin/bash -e + +TEMPDIR=$( mktemp -d ) + +cat << EOF > $TEMPDIR/kubeconfig +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: http://localhost:8080 +users: +- name: local + user: +contexts: +- context: + cluster: local + user: local +EOF + +cleanup () { + docker rm -f $( cat $TEMPDIR/etcd ) + docker rm -f $( cat $TEMPDIR/kube-apiserver ) + rm -rf $TEMPDIR +} + +trap "{ CODE=$?; cleanup ; exit $CODE; }" EXIT + +docker run \ + --cidfile=$TEMPDIR/etcd \ + -d \ + --net=host \ + gcr.io/google_containers/etcd:3.1.10 \ + etcd + +docker run \ + --cidfile=$TEMPDIR/kube-apiserver \ + -d \ + -v $TEMPDIR:/var/run/kube-test:ro \ + --net=host \ + gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 \ + kube-apiserver \ + --etcd-servers=http://localhost:2379 \ + --service-cluster-ip-range=10.0.0.1/16 \ + --insecure-bind-address=0.0.0.0 \ + --insecure-port=8080 + +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/healthz); do + printf '.' + sleep 1 +done +echo "API server ready" + +export DEX_KUBECONFIG=$TEMPDIR/kubeconfig +go test -v -i ./storage/kubernetes +go test -v ./storage/kubernetes diff --git a/storage/kubernetes/client.go b/storage/kubernetes/client.go index 57c975cf..6eb4e258 100644 --- a/storage/kubernetes/client.go +++ b/storage/kubernetes/client.go @@ -157,7 +157,11 @@ func closeResp(r *http.Response) { } func (c *client) get(resource, name string, v interface{}) error { - url := c.urlFor(c.apiVersion, c.namespace, resource, name) + return c.getResource(c.apiVersion, c.namespace, resource, name, v) +} + +func (c *client) getResource(apiVersion, namespace, resource, name string, v interface{}) error { + url := c.urlFor(apiVersion, namespace, resource, name) resp, err := c.client.Get(url) if err != nil { return err diff --git a/storage/kubernetes/storage.go b/storage/kubernetes/storage.go index 56a7dd89..6acf89a0 100644 --- a/storage/kubernetes/storage.go +++ b/storage/kubernetes/storage.go @@ -53,9 +53,9 @@ func (c *Config) Open(logger logrus.FieldLogger) (storage.Storage, error) { // open returns a kubernetes client, initializing the third party resources used // by dex. // -// errOnResources controls if errors creating the resources cause this method to return +// waitForResources controls if errors creating the resources cause this method to return // immediately (used during testing), or if the client will asynchronously retry. -func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, error) { +func (c *Config) open(logger logrus.FieldLogger, waitForResources bool) (*client, error) { if c.InCluster && (c.KubeConfigFile != "") { return nil, errors.New("cannot specify both 'inCluster' and 'kubeConfigFile'") } @@ -86,7 +86,7 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, ctx, cancel := context.WithCancel(context.Background()) if !cli.registerCustomResources(c.UseTPR) { - if errOnResources { + if waitForResources { cancel() return nil, fmt.Errorf("failed creating custom resources") } @@ -110,6 +110,13 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, }() } + if waitForResources { + if err := cli.waitForCRDs(ctx); err != nil { + cancel() + return nil, err + } + } + // If the client is closed, stop trying to create resources. cli.cancel = cancel return cli, nil @@ -122,9 +129,6 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, // It logs all errors, returning true if the resources were created successfully. // // Creating a custom resource does not mean that they'll be immediately available. -// -// TODO(ericchiang): Provide an option to wait for the resources to actually -// be available. func (cli *client) registerCustomResources(useTPR bool) (ok bool) { ok = true length := len(customResourceDefinitions) @@ -164,6 +168,49 @@ func (cli *client) registerCustomResources(useTPR bool) (ok bool) { return ok } +// waitForCRDs waits for all CRDs to be in a ready state, and is used +// by the tests to synchronize before running conformance. +func (cli *client) waitForCRDs(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + for _, crd := range customResourceDefinitions { + for { + err := cli.isCRDReady(crd.Name) + if err == nil { + break + } + + cli.logger.Errorf("checking CRD: %v", err) + + select { + case <-ctx.Done(): + return errors.New("timed out waiting for CRDs to be available") + case <-time.After(time.Millisecond * 100): + } + } + } + return nil +} + +// isCRDReady determines if a CRD is ready by inspecting its conditions. +func (cli *client) isCRDReady(name string) error { + var r k8sapi.CustomResourceDefinition + err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r) + if err != nil { + return fmt.Errorf("get crd %s: %v", name, err) + } + + conds := make(map[string]string) // For debugging, keep the conditions around. + for _, c := range r.Status.Conditions { + if c.Type == k8sapi.Established && c.Status == k8sapi.ConditionTrue { + return nil + } + conds[string(c.Type)] = string(c.Status) + } + return fmt.Errorf("crd %s not ready %#v", name, conds) +} + func (cli *client) Close() error { if cli.cancel != nil { cli.cancel()