From e53bdfabb926781492f0d3f46595c33840f63c89 Mon Sep 17 00:00:00 2001 From: Pavel Borzenkov Date: Fri, 21 Apr 2017 18:51:55 +0300 Subject: [PATCH] storage/sql: initial MySQL storage implementation It will be shared by both Postgres and MySQL configs. Signed-off-by: Pavel Borzenkov --- cmd/dex/config.go | 1 + go.sum | 21 +++++ storage/sql/config.go | 164 ++++++++++++++++++++++++++++++++----- storage/sql/config_test.go | 122 ++++++++++++++++----------- storage/sql/migrate.go | 59 ++++++------- storage/sql/sql.go | 22 +++++ 6 files changed, 292 insertions(+), 97 deletions(-) diff --git a/cmd/dex/config.go b/cmd/dex/config.go index ed1cc9f2..2f3e0c4f 100644 --- a/cmd/dex/config.go +++ b/cmd/dex/config.go @@ -136,6 +136,7 @@ var storages = map[string]func() StorageConfig{ "memory": func() StorageConfig { return new(memory.Config) }, "sqlite3": func() StorageConfig { return new(sql.SQLite3) }, "postgres": func() StorageConfig { return new(sql.Postgres) }, + "mysql": func() StorageConfig { return new(sql.MySQL) }, } // UnmarshalJSON allows Storage to implement the unmarshaler interface to diff --git a/go.sum b/go.sum index 44255005..c79a6941 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Sirupsen/logrus v0.11.0 h1:e67CSCQN/h7Iozb0J2qUoCFldk79UIoPL8FbzsW0r0U= +github.com/Sirupsen/logrus v0.11.0/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= github.com/beevik/etree v0.0.0-20161216042344-4cd0dd976db8 h1:83NNCRw/4bJwVOCZ5NKmRiqbffkDC/B2DFmKZ/EzU0c= github.com/beevik/etree v0.0.0-20161216042344-4cd0dd976db8/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1 h1:OnJHjoVbY69GG4gclp0ngXfywigLhR6rrgUxmxQRWO4= @@ -6,8 +8,13 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 h1:dzj1/xcivGjNPwwifh/dWTczkwcuqsXXFHY1X/TZMtw= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= +github.com/cockroachdb/cockroach-go v0.0.0-20160916181719-31611c0501c8 h1:0qNPJvn7to269Uth4ChEH1SkIUkFlT/SlvWkaXerGgk= +github.com/cockroachdb/cockroach-go v0.0.0-20160916181719-31611c0501c8/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/coreos/dex v2.13.0+incompatible h1:iwTFQFV7PnQ91+vSVhXGWdzEcF4I+U52DgEGDrwGWfk= +github.com/coreos/dex v2.13.0+incompatible/go.mod h1:eWiVFa+I1sIRiwXi4bJMZvd90+H7EhRm/J1Y6Y18AOM= github.com/coreos/etcd v3.2.9+incompatible h1:3TbjfK5+aSRLTU/KgBC1xlgA2dn2ddYQngRqX6HFwlQ= github.com/coreos/etcd v3.2.9+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v0.0.0-20170307191026-be73733bb8cc/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.0.0+incompatible h1:+RStIopZ8wooMx+Vs5Bt8zMXxV1ABl5LbakNExNmZIg= github.com/coreos/go-oidc v2.0.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= @@ -24,10 +31,13 @@ github.com/felixge/httpsnoop v1.0.0 h1:gh8fMGz0rlOv/1WmRZm7OgncIOTsAj21iNJot48om github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/ghodss/yaml v0.0.0-20161020005002-bea76d6a4713 h1:ag3kFMoZZMrEBi4ySTdSWSPz8K7Slu++J9/QXzLBiLk= github.com/ghodss/yaml v0.0.0-20161020005002-bea76d6a4713/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-sql-driver/mysql v0.0.0-20160802113842-0b58b37b664c h1:jdWrt1yXmcXx/g3UFLn3XEqh7DulXBwbbkfUjKlMLWA= +github.com/go-sql-driver/mysql v0.0.0-20160802113842-0b58b37b664c/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v0.0.0-20170331031902-2bba0603135d/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf h1:pFr/u+m8QUBMW/itAczltF3guNRAL7XDs5tD3f6nSD0= github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= @@ -48,6 +58,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.0.0-20160907122059-bcac9884e750 h1:ysDggIU+1XiT58oKSl5C9d+hxeOW7kqeI2n6eZymuMs= github.com/jonboulle/clockwork v0.0.0-20160907122059-bcac9884e750/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -55,6 +66,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb h1:iiMILPl9HQFqdFXIuwfYT73NYtH0KApnCmyF7y5wYhs= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/lib/pq v0.0.0-20160831222520-50761b0867bd h1:1BKcGC7eo9wk4c/y2eqrMj3/Wcmj+ZkMn1JpiCpbEgU= +github.com/lib/pq v0.0.0-20160831222520-50761b0867bd/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc h1:0pifi8wVV/YuUKBDmlH3koJgRVnUJ2RiJQ8ly/1/aJ8= github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v0.0.0-20160907162043-3fb7a0e792ed h1:hhFE3aQaQI9KqFBAfuuRvfNIeqG+ExqgaHwec3Lve6s= @@ -79,10 +92,13 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v0.0.0-20170713114250-a3f95b5c4235 h1:a2XWU6egUZQhD52o2GEKr79zE+OuZmwLybyOQpoqhHQ= github.com/sirupsen/logrus v0.0.0-20170713114250-a3f95b5c4235/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cobra v0.0.0-20160615143614-bc81c21bd0d8 h1:g2skTvb63htNjrbi0JetliK7awRNrZAp1eES43yAZO8= github.com/spf13/cobra v0.0.0-20160615143614-bc81c21bd0d8/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20160610190902-367864438f1b h1:eZZ0QAe7qz2L8dz1t9s6AMlheHgTLP4XcNbCV8HkOkY= github.com/spf13/pflag v0.0.0-20160610190902-367864438f1b/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ugorji/go/codec v0.0.0-20181127175209-856da096dbdf h1:BLcwkDfQ8QPXNXBApZUATvuigovcYPXkHzez80QFGNg= @@ -101,6 +117,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTm golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20151211033651-833a04a10549 h1:imXIGlmpdV8HlMP9DTrSVaxjoffgGbwFZdJl0Ous5dc= golang.org/x/sys v0.0.0-20151211033651-833a04a10549/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170401064109-f4b4367115ec h1:IQbbXMrYo9hsfbt8unKNTFivvnNgGfA4p2HQLJmfrQU= golang.org/x/text v0.0.0-20170401064109-f4b4367115ec/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= @@ -117,8 +135,11 @@ gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 h1:nn6Zav2sOQHCFJHEspya8 gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ldap.v2 v2.3.0/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= +gopkg.in/square/go-jose.v2 v2.0.0 h1:wAvgsVkaEvXffSQ835HNP29ZiYjPj1dh/t2Z6O/pJ3I= +gopkg.in/square/go-jose.v2 v2.0.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.1.8 h1:yECBkTX7ypNaRFILw4trAAYXRLvcGxTeHCBKj/fc8gU= gopkg.in/square/go-jose.v2 v2.1.8/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.0.0-20160301204022-a83829b6f129 h1:RBgb9aPUbZ9nu66ecQNIBNsA7j3mB5h8PNDIfhPjaJg= diff --git a/storage/sql/config.go b/storage/sql/config.go index 8fc715b8..e3262b89 100644 --- a/storage/sql/config.go +++ b/storage/sql/config.go @@ -1,14 +1,18 @@ package sql import ( + "crypto/tls" + "crypto/x509" "database/sql" "fmt" - "net" - "regexp" + "io/ioutil" + "net/url" "strconv" - "strings" "time" + "github.com/Sirupsen/logrus" + "github.com/coreos/dex/storage" + "github.com/go-sql-driver/mysql" "github.com/lib/pq" sqlite3 "github.com/mattn/go-sqlite3" @@ -21,6 +25,12 @@ const ( pgErrUniqueViolation = "23505" // unique_violation ) +const ( + // MySQL error codes + mysqlErrDupEntry = 1062 + mysqlErrDupEntryWithKeyName = 1586 +) + // SQLite3 options for creating an SQL db. type SQLite3 struct { // File to @@ -63,31 +73,29 @@ func (s *SQLite3) open(logger log.Logger) (*conn, error) { } const ( - sslDisable = "disable" - sslRequire = "require" - sslVerifyCA = "verify-ca" - sslVerifyFull = "verify-full" + // postgres SSL modes + pgSSLDisable = "disable" + pgSSLRequire = "require" + pgSSLVerifyCA = "verify-ca" + pgSSLVerifyFull = "verify-full" ) -// PostgresSSL represents SSL options for Postgres databases. -type PostgresSSL struct { - Mode string - CAFile string - // Files for client auth. - KeyFile string - CertFile string -} +const ( + // MySQL SSL modes + mysqlSSLTrue = "true" + mysqlSSLFalse = "false" + mysqlSSLSkipVerify = "skip-verify" + mysqlSSLCustom = "custom" +) -// Postgres options for creating an SQL db. -type Postgres struct { +// NetworkDB contains options common to SQL databases accessed over network. +type NetworkDB struct { Database string User string Password string Host string Port uint16 - SSL PostgresSSL `json:"ssl" yaml:"ssl"` - ConnectionTimeout int // Seconds // database/sql tunables, see @@ -98,6 +106,22 @@ type Postgres struct { ConnMaxLifetime int // Seconds, default: not set } +// SSL represents SSL options for network databases. +type SSL struct { + Mode string + CAFile string + // Files for client auth. + KeyFile string + CertFile string +} + +// Postgres options for creating an SQL db. +type Postgres struct { + NetworkDB + + SSL SSL `json:"ssl" yaml:"ssl"` +} + // Open creates a new storage implementation backed by Postgres. func (p *Postgres) Open(logger log.Logger) (storage.Storage, error) { conn, err := p.open(logger, p.createDataSourceName()) @@ -216,3 +240,105 @@ func (p *Postgres) open(logger log.Logger, dataSourceName string) (*conn, error) } return c, nil } + +// MySQL options for creating a MySQL db. +type MySQL struct { + NetworkDB + + SSL SSL `json:"ssl" yaml:"ssl"` + + // TODO(pborzenkov): used by tests to reduce lock wait timeout. Should + // we make it exported and allow users to provide arbitrary params? + params map[string]string +} + +// Open creates a new storage implementation backed by MySQL. +func (s *MySQL) Open(logger logrus.FieldLogger) (storage.Storage, error) { + conn, err := s.open(logger) + if err != nil { + return nil, err + } + return conn, nil +} + +func (s *MySQL) open(logger logrus.FieldLogger) (*conn, error) { + cfg := mysql.Config{ + User: s.User, + Passwd: s.Password, + DBName: s.Database, + + Timeout: time.Second * time.Duration(s.ConnectionTimeout), + + ParseTime: true, + Params: map[string]string{ + "tx_isolation": "'SERIALIZABLE'", + }, + } + if s.Host != "" { + if s.Host[0] != '/' { + cfg.Net = "tcp" + cfg.Addr = s.Host + } else { + cfg.Net = "unix" + cfg.Addr = s.Host + } + } + if s.SSL.CAFile != "" || s.SSL.CertFile != "" || s.SSL.KeyFile != "" { + if err := s.makeTLSConfig(); err != nil { + return nil, fmt.Errorf("failed to make TLS config: %v", err) + } + cfg.TLSConfig = mysqlSSLCustom + } else { + cfg.TLSConfig = s.SSL.Mode + } + for k, v := range s.params { + cfg.Params[k] = v + } + + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return nil, err + } + + errCheck := func(err error) bool { + sqlErr, ok := err.(*mysql.MySQLError) + if !ok { + return false + } + return sqlErr.Number == mysqlErrDupEntry || + sqlErr.Number == mysqlErrDupEntryWithKeyName + } + + c := &conn{db, flavorMySQL, logger, errCheck} + if _, err := c.migrate(); err != nil { + return nil, fmt.Errorf("failed to perform migrations: %v", err) + } + return c, nil +} + +func (s *MySQL) makeTLSConfig() error { + cfg := &tls.Config{} + if s.SSL.CAFile != "" { + rootCertPool := x509.NewCertPool() + pem, err := ioutil.ReadFile(s.SSL.CAFile) + if err != nil { + return err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return fmt.Errorf("failed to append PEM") + } + cfg.RootCAs = rootCertPool + } + if s.SSL.CertFile != "" && s.SSL.KeyFile != "" { + clientCert := make([]tls.Certificate, 0, 1) + certs, err := tls.LoadX509KeyPair(s.SSL.CertFile, s.SSL.KeyFile) + if err != nil { + return err + } + clientCert = append(clientCert, certs) + cfg.Certificates = clientCert + } + + mysql.RegisterTLSConfig(mysqlSSLCustom, cfg) + return nil +} diff --git a/storage/sql/config_test.go b/storage/sql/config_test.go index d823e099..d8136496 100644 --- a/storage/sql/config_test.go +++ b/storage/sql/config_test.go @@ -32,15 +32,16 @@ func withTimeout(t time.Duration, f func()) { } func cleanDB(c *conn) error { - _, err := c.Exec(` - delete from client; - delete from auth_request; - delete from auth_code; - delete from refresh_token; - delete from keys; - delete from password; - `) - return err + tables := []string{"client", "auth_request", "auth_code", + "refresh_token", "keys", "password"} + + for _, tbl := range tables { + _, err := c.Exec("delete from " + tbl) + if err != nil { + return err + } + } + return nil } var logger = &logrus.Logger{ @@ -49,23 +50,39 @@ var logger = &logrus.Logger{ Level: logrus.DebugLevel, } -func TestSQLite3(t *testing.T) { +type opener interface { + open(logrus.FieldLogger) (*conn, error) +} + +func testDB(t *testing.T, o opener, withTransactions bool) { + // t.Fatal has a bad habbit of not actually printing the error + fatal := func(i interface{}) { + fmt.Fprintln(os.Stdout, i) + t.Fatal(i) + } + newStorage := func() storage.Storage { - // NOTE(ericchiang): In memory means we only get one connection at a time. If we - // ever write tests that require using multiple connections, for instance to test - // transactions, we need to move to a file based system. - s := &SQLite3{":memory:"} - conn, err := s.open(logger) + conn, err := o.open(logger) if err != nil { - fmt.Fprintln(os.Stdout, err) - t.Fatal(err) + fatal(err) + } + if err := cleanDB(conn); err != nil { + fatal(err) } return conn } - - withTimeout(time.Second*10, func() { + withTimeout(time.Minute*1, func() { conformance.RunTests(t, newStorage) }) + if withTransactions { + withTimeout(time.Minute*1, func() { + conformance.RunTransactionTests(t, newStorage) + }) + } +} + +func TestSQLite3(t *testing.T) { + testDB(t, &SQLite3{":memory:"}, false) } func getenv(key, defaultVal string) string { @@ -186,37 +203,42 @@ func TestPostgres(t *testing.T) { if host == "" { t.Skipf("test environment variable %q not set, skipping", testPostgresEnv) } - p := Postgres{ - Database: getenv("DEX_POSTGRES_DATABASE", "postgres"), - User: getenv("DEX_POSTGRES_USER", "postgres"), - Password: getenv("DEX_POSTGRES_PASSWORD", "postgres"), - Host: host, - SSL: PostgresSSL{ - Mode: sslDisable, // Postgres container doesn't support SSL. + p := &Postgres{ + NetworkDB: NetworkDB{ + Database: getenv("DEX_POSTGRES_DATABASE", "postgres"), + User: getenv("DEX_POSTGRES_USER", "postgres"), + Password: getenv("DEX_POSTGRES_PASSWORD", "postgres"), + Host: host, + ConnectionTimeout: 5, + }, + SSL: SSL{ + Mode: pgSSLDisable, // Postgres container doesn't support SSL. }, - ConnectionTimeout: 5, } - - // t.Fatal has a bad habbit of not actually printing the error - fatal := func(i interface{}) { - fmt.Fprintln(os.Stdout, i) - t.Fatal(i) - } - - newStorage := func() storage.Storage { - conn, err := p.open(logger, p.createDataSourceName()) - if err != nil { - fatal(err) - } - if err := cleanDB(conn); err != nil { - fatal(err) - } - return conn - } - withTimeout(time.Minute*1, func() { - conformance.RunTests(t, newStorage) - }) - withTimeout(time.Minute*1, func() { - conformance.RunTransactionTests(t, newStorage) - }) + testDB(t, p, true) +} + +const testMySQLEnv = "DEX_MYSQL_HOST" + +func TestMySQL(t *testing.T) { + host := os.Getenv(testMySQLEnv) + if host == "" { + t.Skipf("test environment variable %q not set, skipping", testMySQLEnv) + } + s := &MySQL{ + NetworkDB: NetworkDB{ + Database: getenv("DEX_MYSQL_DATABASE", "mysql"), + User: getenv("DEX_MYSQL_USER", "mysql"), + Password: getenv("DEX_MYSQL_PASSWORD", ""), + Host: host, + ConnectionTimeout: 5, + }, + SSL: SSL{ + Mode: mysqlSSLFalse, + }, + params: map[string]string{ + "innodb_lock_wait_timeout": "3", + }, + } + testDB(t, s, true) } diff --git a/storage/sql/migrate.go b/storage/sql/migrate.go index 6341037a..e30629e7 100644 --- a/storage/sql/migrate.go +++ b/storage/sql/migrate.go @@ -38,8 +38,10 @@ func (c *conn) migrate() (int, error) { migrationNum := n + 1 m := migrations[n] - if _, err := tx.Exec(m.stmt); err != nil { - return fmt.Errorf("migration %d failed: %v", migrationNum, err) + for i := range m.stmts { + if _, err := tx.Exec(m.stmts[i]); err != nil { + return fmt.Errorf("migration %d statement %d failed: %v", migrationNum, i+1, err) + } } q := `insert into migrations (num, at) values ($1, now());` @@ -61,14 +63,14 @@ func (c *conn) migrate() (int, error) { } type migration struct { - stmt string + stmts []string // TODO(ericchiang): consider adding additional fields like "forDrivers" } // All SQL flavors share migration strategies. var migrations = []migration{ { - stmt: ` + stmts: []string{` create table client ( id text not null primary key, secret text not null, @@ -77,8 +79,8 @@ var migrations = []migration{ public boolean not null, name text not null, logo_url text not null - ); - + );`, + ` create table auth_request ( id text not null primary key, client_id text not null, @@ -101,8 +103,8 @@ var migrations = []migration{ connector_data bytea, expiry timestamptz not null - ); - + );`, + ` create table auth_code ( id text not null primary key, client_id text not null, @@ -120,8 +122,8 @@ var migrations = []migration{ connector_data bytea, expiry timestamptz not null - ); - + );`, + ` create table refresh_token ( id text not null primary key, client_id text not null, @@ -136,15 +138,15 @@ var migrations = []migration{ connector_id text not null, connector_data bytea - ); - + );`, + ` create table password ( email text not null primary key, hash bytea not null, username text not null, user_id text not null - ); - + );`, + ` -- keys is a weird table because we only ever expect there to be a single row create table keys ( id text not null primary key, @@ -152,39 +154,40 @@ var migrations = []migration{ signing_key bytea not null, -- JSON object signing_key_pub bytea not null, -- JSON object next_rotation timestamptz not null - ); - - `, + );`, + }, }, { - stmt: ` + stmts: []string{` alter table refresh_token - add column token text not null default ''; + add column token text not null default '';`, + ` alter table refresh_token - add column created_at timestamptz not null default '0001-01-01 00:00:00 UTC'; + add column created_at timestamptz not null default '0001-01-01 00:00:00 UTC';`, + ` alter table refresh_token - add column last_used timestamptz not null default '0001-01-01 00:00:00 UTC'; - `, + add column last_used timestamptz not null default '0001-01-01 00:00:00 UTC';`, + }, }, { - stmt: ` + stmts: []string{` create table offline_session ( user_id text not null, conn_id text not null, refresh bytea not null, PRIMARY KEY (user_id, conn_id) - ); - `, + );`, + }, }, { - stmt: ` + stmts: []string{` create table connector ( id text not null primary key, type text not null, name text not null, resource_version text not null, config bytea - ); - `, + );`, + }, }, } diff --git a/storage/sql/sql.go b/storage/sql/sql.go index 26e65f7d..de1d1463 100644 --- a/storage/sql/sql.go +++ b/storage/sql/sql.go @@ -83,6 +83,28 @@ var ( {regexp.MustCompile(`\bnow\(\)`), "date('now')"}, }, } + + flavorMySQL = flavor{ + queryReplacers: []replacer{ + {bindRegexp, "?"}, + // Translate types. + {matchLiteral("bytea"), "blob"}, + {matchLiteral("timestamptz"), "datetime(3)"}, + // MySQL doesn't support indicies on text fields w/o + // specifying key length. Use varchar instead (768 is + // the max key length for InnoDB with 4k pages). + {matchLiteral("text"), "varchar(768)"}, + // Quote keywords and reserved words used as identifiers. + {regexp.MustCompile(`\b(keys)\b`), "`$1`"}, + // Change default timestamp to fit datetime. + {regexp.MustCompile(`0001-01-01 00:00:00 UTC`), "1000-01-01 00:00:00"}, + }, + } + + // Not tested. + flavorCockroach = flavor{ + executeTx: crdb.ExecuteTx, + } ) func (f flavor) translate(query string) string {