From 309b33d05a2573f949f2fdd65fb991e24f73ed9d Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Wed, 18 Dec 2019 17:00:57 +0100 Subject: [PATCH 1/3] Rewrite kubernetes storage test --- Makefile | 17 ++++- go.mod | 1 + go.sum | 13 ++++ storage/kubernetes/storage_test.go | 115 +++++++++++++++++++---------- 4 files changed, 107 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 2c041408..81a6dbcd 100644 --- a/Makefile +++ b/Makefile @@ -41,12 +41,25 @@ revendor: @go mod vendor -v @go mod verify -test: +test: bin/test/kube-apiserver bin/test/etcd @go test -v ./... -testrace: +testrace: bin/test/kube-apiserver bin/test/etcd @go test -v --race ./... +export TEST_ASSET_KUBE_APISERVER=$(abspath bin/test/kube-apiserver) +export TEST_ASSET_ETCD=$(abspath bin/test/etcd) + +bin/test/kube-apiserver: + @mkdir -p bin/test + curl -L https://storage.googleapis.com/k8s-c10s-test-binaries/kube-apiserver-$(shell uname)-x86_64 > bin/test/kube-apiserver + chmod +x bin/test/kube-apiserver + +bin/test/etcd: + @mkdir -p bin/test + curl -L https://storage.googleapis.com/k8s-c10s-test-binaries/etcd-$(shell uname)-x86_64 > bin/test/etcd + chmod +x bin/test/etcd + bin/golangci-lint: bin/golangci-lint-${GOLANGCI_VERSION} @ln -sf golangci-lint-${GOLANGCI_VERSION} bin/golangci-lint bin/golangci-lint-${GOLANGCI_VERSION}: diff --git a/go.mod b/go.mod index 99a51004..7f489d7c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.3.1 + sigs.k8s.io/testing_frameworks v0.1.2 ) go 1.13 diff --git a/go.sum b/go.sum index 73f3b2e1..a82bad50 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -172,9 +173,11 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -273,6 +276,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -300,6 +304,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -317,6 +324,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -363,6 +371,8 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= @@ -373,6 +383,7 @@ gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -385,3 +396,5 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= +sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= diff --git a/storage/kubernetes/storage_test.go b/storage/kubernetes/storage_test.go index 27d65416..ea471427 100644 --- a/storage/kubernetes/storage_test.go +++ b/storage/kubernetes/storage_test.go @@ -1,39 +1,104 @@ package kubernetes import ( - "fmt" + "io/ioutil" "os" + "strings" "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + "sigs.k8s.io/testing_frameworks/integration" "github.com/dexidp/dex/storage" "github.com/dexidp/dex/storage/conformance" ) -const testKubeConfigEnv = "DEX_KUBECONFIG" +const kubeconfigTemplate = `apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: SERVERURL +users: +- name: local + user: +contexts: +- context: + cluster: local + user: local +` -func TestLoadClient(t *testing.T) { - loadClient(t) +func TestStorage(t *testing.T) { + if os.Getenv("TEST_ASSET_KUBE_APISERVER") == "" || os.Getenv("TEST_ASSET_ETCD") == "" { + t.Skip("control plane binaries are missing") + } + + suite.Run(t, new(StorageTestSuite)) } -func loadClient(t *testing.T) *client { +type StorageTestSuite struct { + suite.Suite + + controlPlane *integration.ControlPlane + + client *client +} + +func (s *StorageTestSuite) SetupSuite() { + s.controlPlane = &integration.ControlPlane{} + + err := s.controlPlane.Start() + s.Require().NoError(err) +} + +func (s *StorageTestSuite) TearDownSuite() { + s.controlPlane.Stop() +} + +func (s *StorageTestSuite) SetupTest() { + f, err := ioutil.TempFile("", "dex-kubeconfig-*") + s.Require().NoError(err) + defer f.Close() + + _, err = f.WriteString(strings.ReplaceAll(kubeconfigTemplate, "SERVERURL", s.controlPlane.APIURL().String())) + s.Require().NoError(err) + config := Config{ - KubeConfigFile: os.Getenv(testKubeConfigEnv), - } - if config.KubeConfigFile == "" { - t.Skipf("test environment variable %q not set, skipping", testKubeConfigEnv) + KubeConfigFile: f.Name(), } + logger := &logrus.Logger{ Out: os.Stderr, Formatter: &logrus.TextFormatter{DisableColors: true}, Level: logrus.DebugLevel, } - s, err := config.open(logger, true) - if err != nil { - t.Fatal(err) + + client, err := config.open(logger, true) + s.Require().NoError(err) + + s.client = client +} + +func (s *StorageTestSuite) TestStorage() { + newStorage := func() storage.Storage { + for _, resource := range []string{ + resourceAuthCode, + resourceAuthRequest, + resourceClient, + resourceRefreshToken, + resourceKeys, + resourcePassword, + } { + if err := s.client.deleteAll(resource); err != nil { + s.T().Fatalf("delete all %q failed: %v", resource, err) + } + } + return s.client } - return s + + conformance.RunTests(s.T(), newStorage) + conformance.RunTransactionTests(s.T(), newStorage) } func TestURLFor(t *testing.T) { @@ -83,27 +148,3 @@ func TestURLFor(t *testing.T) { } } } - -func TestStorage(t *testing.T) { - client := loadClient(t) - newStorage := func() storage.Storage { - for _, resource := range []string{ - resourceAuthCode, - resourceAuthRequest, - resourceClient, - resourceRefreshToken, - resourceKeys, - resourcePassword, - } { - if err := client.deleteAll(resource); err != nil { - // Fatalf sometimes doesn't print the error message. - fmt.Fprintf(os.Stderr, "delete all %q failed: %v\n", resource, err) - t.Fatalf("delete all %q failed: %v", resource, err) - } - } - return client - } - - conformance.RunTests(t, newStorage) - conformance.RunTransactionTests(t, newStorage) -} From e0c58d54496e8ce33918d04013d9e8378e87a76c Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Wed, 18 Dec 2019 17:02:05 +0100 Subject: [PATCH 2/3] Remove old kubernetes storage test flow --- .github/workflows/ci.yml | 3 --- .travis.yml | 1 - scripts/test-k8s.sh | 56 ---------------------------------------- 3 files changed, 60 deletions(-) delete mode 100755 scripts/test-k8s.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46c9c928..8d4e906d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,9 +68,6 @@ jobs: DEX_KEYSTONE_ADMIN_USER: demo DEX_KEYSTONE_ADMIN_PASS: DEMO_PASS - - name: Run Kubernetes tests - run: ./scripts/test-k8s.sh - - name: Run linter run: make lint diff --git a/.travis.yml b/.travis.yml index d8ef066e..6c820183 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,6 @@ install: script: - make testall - - ./scripts/test-k8s.sh - make verify-proto # Ensure proto generation doesn't depend on external packages. notifications: diff --git a/scripts/test-k8s.sh b/scripts/test-k8s.sh deleted file mode 100755 index 86127be1..00000000 --- a/scripts/test-k8s.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -e - -TEMPDIR=$( mktemp -d ) - -cat << EOF > $TEMPDIR/kubeconfig -apiVersion: v1 -kind: Config -clusters: -- name: local - cluster: - server: http://localhost:8080 -users: -- name: local - user: -contexts: -- context: - cluster: local - user: local -EOF - -cleanup () { - docker rm -f $( cat $TEMPDIR/etcd ) - docker rm -f $( cat $TEMPDIR/kube-apiserver ) - rm -rf $TEMPDIR -} - -trap "{ CODE=$?; cleanup ; exit $CODE; }" EXIT - -docker run \ - --cidfile=$TEMPDIR/etcd \ - -d \ - --net=host \ - gcr.io/google_containers/etcd:3.1.10 \ - etcd - -docker run \ - --cidfile=$TEMPDIR/kube-apiserver \ - -d \ - -v $TEMPDIR:/var/run/kube-test:ro \ - --net=host \ - gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 \ - kube-apiserver \ - --etcd-servers=http://localhost:2379 \ - --service-cluster-ip-range=10.0.0.1/16 \ - --insecure-bind-address=0.0.0.0 \ - --insecure-port=8080 - -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/healthz); do - printf '.' - sleep 1 -done -echo "API server ready" - -export DEX_KUBECONFIG=$TEMPDIR/kubeconfig -go test -v -i ./storage/kubernetes -go test -v ./storage/kubernetes From 3fb85ab0094b94f1348a9e0697279f93fa3cba96 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Wed, 18 Dec 2019 17:03:46 +0100 Subject: [PATCH 3/3] Remove instructions for kubernetes tests from docs --- Documentation/dev-integration-tests.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Documentation/dev-integration-tests.md b/Documentation/dev-integration-tests.md index c9c84d2d..1b694580 100644 --- a/Documentation/dev-integration-tests.md +++ b/Documentation/dev-integration-tests.md @@ -1,21 +1,5 @@ # Running integration tests -## Kubernetes - -Kubernetes tests run against a Kubernetes API server, and are enabled by the `DEX_KUBECONFIG` environment variable: - -``` -$ export DEX_KUBECONFIG=~/.kube/config -$ go test -v -i ./storage/kubernetes -$ go test -v ./storage/kubernetes -``` - -These tests can be executed locally using docker by running the following script: - -``` -$ ./scripts/test-k8s.sh -``` - ## Postgres Running database tests locally requires: