From 3bbafaf3fe108f81cc4c299224a2e2a479430187 Mon Sep 17 00:00:00 2001 From: Bobby Rullo Date: Fri, 21 Aug 2015 16:39:46 -0700 Subject: [PATCH 1/4] README.md/Documentation: Get Docs up-to-date. * Fix up README * Create getting started guide. * Start dev guide * Start deploy guide --- Documentation/deploy-guide.md | 9 ++ Documentation/dev-guide.md | 45 ++++++++++ Documentation/getting-started.md | 140 +++++++++++++++++++++++++++++++ README.md | 124 ++++++--------------------- 4 files changed, 220 insertions(+), 98 deletions(-) create mode 100644 Documentation/deploy-guide.md create mode 100644 Documentation/dev-guide.md create mode 100644 Documentation/getting-started.md diff --git a/Documentation/deploy-guide.md b/Documentation/deploy-guide.md new file mode 100644 index 00000000..9d284208 --- /dev/null +++ b/Documentation/deploy-guide.md @@ -0,0 +1,9 @@ +# Deploying + +Generate systemd unit files by injecting secrets into the unit file templates located in: `./static/...`. + +``` +source /prod/dex.env.txt +./build-units +``` + diff --git a/Documentation/dev-guide.md b/Documentation/dev-guide.md new file mode 100644 index 00000000..9519bdca --- /dev/null +++ b/Documentation/dev-guide.md @@ -0,0 +1,45 @@ +# Dev Guide + + +## Building + +To build using the go binary on your host, use the `./build` script. + +You can also use a copy of `go` hosted inside a docker container if you prefix your command with `go-docker`, as in: `./go-docker ./build` + +## Docker Build and Push + +Once binaries are compiled you can build and push a dex image to quay.io. Before doing this step binaries must be built above using one of the build tools. + +``` +export DOCKER_USER=<> +export DOCKER_PASSWORD=<> +./build-docker-push +``` + +## Rebuild API from JSON schema + +Go API bindings are generated from a JSON Discovery file. +To regenerate run: + +``` +./schema/generator +``` + +For updating generator dependencies see docs in: `schema/generator_import.go`. + +## Runing Tests + +Run all tests: `./test` + +Single package only: `PKG= ./test` + +Functional tests: `./test-functional` + +Run with docker: + +``` +./go-docker ./test +./go-docker ./test-functional +``` + diff --git a/Documentation/getting-started.md b/Documentation/getting-started.md new file mode 100644 index 00000000..48eb753e --- /dev/null +++ b/Documentation/getting-started.md @@ -0,0 +1,140 @@ +# Getting Started + + +# Introduction + +In this document we'll stand up the full dex stack on a single machine. This should demonstrate all the moving parts involved in a dex installation, but is not appropriate for production deployment. Please see the [deployment guide][deployment-guide] for information on production dex setups. + +[deployment-guide]: https://github.com/coreos/dex/blob/master/Documentation/deployment-guide.md + +We'll also start the example web app, so we can try registering and logging in. + +# Pre-requisites + +Before continuing, you must have the following installed on your system: + +* Go 1.4 or greater +* Postgres 9.0 or greater + +In addition, if you wish to try out authenticating against Google's OIDC backend, you must have a new client registered with Google: + +* Go to https://console.developers.google.com/project and create a new project. +* Click on credentials, and ask to set up an OAuth 2 client ID. You'll then need to give your project a name. +* Back at the "Create Client ID" screen, choose "Web Application" and enter `http://127.0.0.1:5556/auth/google/callback` for your Redirect URI. + +# Create Database + +`createdb dex_db` + +Let's store the connection string in a shell variable: + +`DEX_DB_URL=postgres://localhost/dex_db?sslmode=disable` + +# Building + +The build script will build all dex components. + +`./build` + +# Generate a Secret Symmetric Key + +dex needs a 32 byte base64-encoded key which will be used to encrypt the private keys in the database. A good way to generate the key is to read from /dev/random: + +`DEX_KEY_SECRET=$(dd if=/dev/random bs=1 count=32 2>/dev/null | base64)` + +# Start the overlord + +The overlord is responsible for creating and rotating keys and some other adminsitrative tasks. In addition, the overlord is responsible for creating the necessary database tables (and when you update, performing schema migrations), so it must be started before we do anything else. Debug logging is turned on so we can see more of what's going on. Start it up. + +`./bin/dex-overlord --db-url=$DEX_DB_URL --key-secret=$DEX_KEY_SECRET --log-debug=true &` + +## Environment Variables. + +Note that parameters can be passed as flags or environment variables to dex components; an equivalent start with environment variables would be: + +``` +export DEX_OVERLORD_DB_URL=$DEX_DB_URL +export DEX_OVERLORD_KEY_SECRETS=$DEX_KEY_SECRET +export DEX_OVERLORD_LOG_DEBUG=true +./bin/dex-overlord & +``` + +# Start the dex-worker + +Now start the worker: + +`./bin/dex-worker --db-url=$DEX_DB_URL --key-secrets=$DEX_KEY_SECRET --email-cfg=static/fixtures/emailer.json.sample --log-debug=true &` + +Now you have a worker which you can authenticate against, listening on `http://0.0.0.0:5556`, which is the default. Note that the default issuer URL (which can be changed on --issuerURL) is `http://127.0.0.1:5556`. The issuer URL is the base URL (i.e. no query or fragments) uniquely identifying your dex installation. + +Note: the issuer URL MUST have an `https` scheme in production to meet spec compliance and to be considered reasonably secure. + +# Set up Connectors + +The worker and overlord are up and running, but we need to tell dex what connectors we want to use to authenticate. For this case we'll set up a local connector, where dex manages credentials and provides a UI for authentication, and a Google OIDC connector. + +If you prefer to use the Google OIDC Identity Provider (IdP), just omit the second entry in the JSON connector list. Note that you must replace DEX_GOOGLE_CLIENT_{ID,SECRET} with the client ID and client Secret you got when you registered your project with the Google developer console. +``` +cat << EOF > /tmp/dex_connectors.json +[ + { + "type": "local", + "id": "local" + }, + { + "type": "oidc", + "id": "google", + "issuerURL": "https://accounts.google.com", + "clientID": "$DEX_GOOGLE_CLIENT_ID", + "clientSecret": "$DEX_GOOGLE_CLIENT_SECRET", + "trustedEmailProvider": true + } +] +EOF +./bin/dexctl -db-url=$DEX_DB_URL set-connector-configs /tmp/dex_connectors.json +``` + +One thing to note here that's a bit confusing here is that in the case of the Google OIDC connector, dex is the client and Google is the IdP, but when you're dealing with your own apps that want to authenticate against dex, your app is the client and dex is the IdP. + +# Register a Client + +Like all OAuth2/OIDC IdPs, clients must be registered with the IdP (dex), along with their valid redirect URLS. + +New clients can be registered with the dexctl CLI tool: +``` +eval "$(./bin/dexctl -db-url=$DEX_DB_URL new-client http://127.0.0.1:5555/callback)" +``` + +The output of this command is eval'able if you are in bash, and sets the following shell variables: + +``` +DEX_APP_CLIENT_ID +DEX_APP_CLIENT_SECRET +DEX_APP_REDIRECTURL +``` + +# Start the Example Web App + +The included example app demonstrates registering and authenticating with dex. Start it up: + +``` +./bin/example-app --client-id=$DEX_APP_CLIENT_ID --client-secret=$DEX_APP_CLIENT_SECRET --discovery=http://127.0.0.1:5556 & +``` + +# Authenticate with dex! + +Go to `127.0.0.1:5555`, and click "register"; choose either "Google", if you have a Google Account and would like to use that to authenticate. Otherwise, choose "local". + +If you chose Google, enter your credentials (if you are not logged into Google) and click through the authorization screen. If you chose "local", enter a name and password and submit. + +After registering you should end up back at the example app, where it will display the claims returned by dex. + +# Verify Your Email + +If you registered with Google, your email address is already verified, and this should be reflected by the presence of an `email_verified` claim. Otherwise, you need to verify your email address. + +In a fully configured production environment an email provider will be set up so that dex can email users email verification links (amongst other things); in this setup, we are using the `FakeEmailer` email provider which simply outputs to stdout. Look for the "Welcome to Dex!" message in your console and copy the link that follows it, and then paste it in your browser; you should end up back at the example app page that displays claims, but this time you'll see a tru `email_verified` claim. + +# Standup Dev Script + +A script which does almost everything in this guide exists at `contrib/standup-db.sh`. Read the comments inside before attemping to run it - it requires a little setup beforehand. diff --git a/README.md b/README.md index 71e7c074..dda9e769 100644 --- a/README.md +++ b/README.md @@ -3,34 +3,48 @@ dex [![Docker Repository on Quay.io](https://quay.io/repository/coreos/dex/status?token=5a9732e4-53d6-4419-b56b-9f784f7f9233 "Docker Repository on Quay.io")](https://quay.io/repository/coreos/dex) -dex is a federated identity management service. -It provides OpenID Connect (OIDC) to users, while it proxies to multiple remote identity providers (IdP) to drive actual authentication. +dex is a federated identity management service. It provides OpenID Connect (OIDC) to users, and can proxy to multiple remote identity providers (IdP) to drive actual authentication, as well as managing local username/password credentials. + We named the project 'dex' beceause it is a central index of users that other pieces of software can authenticate against. + ## Architecture dex consists of multiple components: - **dex-worker** is the primary server component of dex - - host a user-facing API that drives the OIDC protocol + - host a user-facing API that drives the OIDC protocol - proxy to remote identity providers via "connectors" + - provides an API for administrators to manage users. - **dex-overlord** is an auxiliary process responsible for two things: - rotation of keys used by the workers to sign identity tokens - garbage collection of stale data in the database + - provides an API for bootstrapping the system. - **dexctl** is CLI tool used to manage an dex deployment - configure identity provider connectors - administer OIDC client identities +- **database**; a database is used to for persistent storage for keys, users, + OAuth sessions and other data. Currently Postgres is the only supported + database. A typical dex deployment consists of N dex-workers behind a load balanacer, and one dex-overlord. The dex-workers directly handle user requests, so the loss of all workers can result in service downtime. The single dex-overlord runs its tasks periodically, so it does not need to maintain 100% uptime. +## Who Should Use AuthD? + + **TODO** + +## Similar Software + + **TODO** + ## Connectors -Remote IdPs could implement any auth-N protocol. -*connectors* contain protocol-specific logic and are used to communicate with remote IdPs. -Possible examples of connectors could be: OIDC, LDAP, Local Memory, Basic Auth, etc. -dex ships with an OIDC connector, and a basic "local" connector for in-memory testing purposes. +Remote IdPs could implement any auth-N protocol. *Connectors* contain protocol-specific logic and are used to communicate with remote IdPs. Possible examples of connectors could be: OIDC, LDAP, Local credentials, Basic Auth, etc. + +dex ships with an OIDC connector, useful for authenticating with services like Google and Salesforce (or even other dex instances!) and a "local" connector, in which dex itself presents a UI for users to authenticate via dex-stored credentials. + Future connectors can be developed and added as future interoperability requirements emerge. ## Relevant Specifications @@ -52,100 +66,14 @@ OpenID Connect (OIDC) is broken up into several specifications. The following (a - https://accounts.google.com/.well-known/openid-configuration - https://login.salesforce.com/.well-known/openid-configuration -# Building +# Next steps: -## With Host Go Environment +If you want to try out dex quickly with a single process and no database (do *not* run this way in production!) take a look at the [dev guide][dev-guide]. -`./build` +For running the full stack check out the [getting started guide][getting-started]. -## With Docker - -`./go-docker ./build` - -## Docker Build and Push - -Binaries must be compiled first. -Builds a docker image and pushes it to the quay repo. -The image is tagged with the git sha and 'latest'. - -``` -export QUAY_USER=xxx -export QUAY_PASSWORD=yyy -./build-docker-push -``` - -## Rebuild API from JSON schema - -Go API bindings are generated from a JSON Discovery file. -To regenerate run: - -``` -./schema/generator -``` - -For updating generator dependencies see docs in: `schema/generator_import.go`. - -## Runing Tests - -Run all tests: `./test` - -Single package only: `PKG= ./test` - -Functional tests: `./test-functional` - -Run with docker: - -``` -./go-docker ./test -./go-docker ./test-functional -``` - -# Running - -Run the main dex server: - -After building, run `./bin/dex` and provider the required arguments. -Additionally start `./bin/dex-overlord` for key rotation and database garbage collection. - -# Deploying - -Generate systemd unit files by injecting secrets into the unit file templates located in: `./static/...`. - -``` -source /prod/dex.env.txt -./build-units -``` - -Resulting unit files are output to: `./deploy` - -# Registering Clients - -Like all OAuth2 servers clients must be registered with a callback url. -New clients can be registered with the dexctl CLI tool: -``` -dexctl --db-url=postgres://localhost/auth?sslmode=disable new-client http://example.com/auth/callback -``` - -The tool will print the `client-id` and `client-secret` to stdout; you must save these for use in your client application. The output of this command is "KEY=VALUE" format, so If you `eval` it in your shell, the relevant variables are available to use. - -Note that for the initial invocation of `dexctl` you need to provide a DSN URL to create a new-client. Once you have created this initial client, you can use its client-id and client-secret as credentials to dexctl, and make requests via the HTTP API instead of the DB: - -``` -dexctl --endpoint=http://your-issuer-url --client-id=your_client_id --client-secret=your_client_secret new-client -``` - -or, if you want to go the eval route: -``` -eval "$(dexctl --endpoint=http://your-issuer-url --client-id=your_client_id --client-secret=your_client_secret new-client)" -``` - -The latter form makes the variables `DEX_APP_CLIENT_ID`, `DEX_APP_CLIENT_SECRET` and `DEX_APP_REDIRECTURL_0` available to your shell. - -This will allow you to create new clients from machines that cannot hit the database. - -# Standup Dev Script - -A script which will create a database, create a client, start an overlord and a worker and start the example app exists at `contrib/standup-db.sh`. +[getting-started]: https://github.com/coreos/dex/blob/master/Documentation/getting-started.md +[dev-guide]: https://github.com/coreos/dex/blob/master/Documentation/dev-guide.md # Coming Soon From 5ecad3470bb94b3b4aacf0564d3a5de134be7d70 Mon Sep 17 00:00:00 2001 From: Bobby Rullo Date: Mon, 24 Aug 2015 15:16:30 -0700 Subject: [PATCH 2/4] Documentation: bring dev-guide up to date. --- Documentation/dev-guide.md | 39 +++++++++++++++++++++++++------- Documentation/getting-started.md | 6 +++-- README.md | 2 +- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/Documentation/dev-guide.md b/Documentation/dev-guide.md index 9519bdca..412d491b 100644 --- a/Documentation/dev-guide.md +++ b/Documentation/dev-guide.md @@ -1,11 +1,20 @@ # Dev Guide +## No DB mode + +When you are working on dex it's convenient to use the "--no-db" flag, which starts up Dex in a mode which uses an in-memory datastore for persistence. It also does not rotate keys, so no overlord is required. + +In this mode you provide the binary with paths to files to connectors and users - there are example files you can use inside of `static/fixtures`, named "connectors.json.sample" and "users.json.sample" respectively. If you rename these to the equivalent without the ".sample", the defaults point to this location, making starting dex as simple as: + +`./bin/dex-worker --no-db` + +*Do not use this flag in production* - it's not thread safe and data is destroyed when the process dies. In addition, there is no key rotation. ## Building To build using the go binary on your host, use the `./build` script. -You can also use a copy of `go` hosted inside a docker container if you prefix your command with `go-docker`, as in: `./go-docker ./build` +You can also use a copy of `go` hosted inside a Docker container if you prefix your command with `go-docker`, as in: `./go-docker ./build` ## Docker Build and Push @@ -17,6 +26,8 @@ export DOCKER_PASSWORD=<> ./build-docker-push ``` +By default the script pushes to `quay.io/coreos/dex`; if you want to push to a different repository, override the `DOCKER_REGISTRY` and `DOCKER_REPO` environment variables. + ## Rebuild API from JSON schema Go API bindings are generated from a JSON Discovery file. @@ -28,18 +39,30 @@ To regenerate run: For updating generator dependencies see docs in: `schema/generator_import.go`. -## Runing Tests +## Running Tests -Run all tests: `./test` +To run all tests (except functional) use the `./test` script; -Single package only: `PKG= ./test` +If you want to test a single package only, use `PKG= ./test` -Functional tests: `./test-functional` +The functional tests require a database; create a database (eg. `createdb dex_func_test`) and then pass it as an environment variable to the functional test script, eg. `DEX_TEST_DSN=postgres://localhost/dex_func_test?sslmode=disable ./test-functional` + +To run these tests with Docker is a little trickier; you need to have a container running Postgres, and then you need to link that container to the container running your tests: -Run with docker: ``` -./go-docker ./test -./go-docker ./test-functional +# Run the Postgres docker container, which creates a db called "postgres" +docker run --name dex_postgres -d postgres + +# The host name in the DSN is "postgres"; that works because that is what we +# will alias the link as, which causes Docker to modify /etc/hosts with a "postgres" +# entry. +export DEX_TEST_DSN=postgres://postgres@postgres/postgres?sslmode=disable + +# Run the test container, linking it to the Postgres container. +DOCKER_LINKS=dex_postgres:postgres DOCKER_ENV=DEX_TEST_DSN ./go-docker ./test-functional docker + +# Remove the container after the tests are run. +rm -f dex_postgres ``` diff --git a/Documentation/getting-started.md b/Documentation/getting-started.md index 48eb753e..28c9ca2c 100644 --- a/Documentation/getting-started.md +++ b/Documentation/getting-started.md @@ -14,7 +14,7 @@ We'll also start the example web app, so we can try registering and logging in. Before continuing, you must have the following installed on your system: * Go 1.4 or greater -* Postgres 9.0 or greater +* Postgres 9.0 or greater (this guide also assumes that Postgres is up and running) In addition, if you wish to try out authenticating against Google's OIDC backend, you must have a new client registered with Google: @@ -42,11 +42,13 @@ dex needs a 32 byte base64-encoded key which will be used to encrypt the private `DEX_KEY_SECRET=$(dd if=/dev/random bs=1 count=32 2>/dev/null | base64)` +The dex overlord and workers allow multiple key secrets (separated by commas) to be passed but only the first will be used to encrypt data; the rest are there for decryption only; this scheme allows for the rotation of keys without downtime (assuming a rolling restart of workers). + # Start the overlord The overlord is responsible for creating and rotating keys and some other adminsitrative tasks. In addition, the overlord is responsible for creating the necessary database tables (and when you update, performing schema migrations), so it must be started before we do anything else. Debug logging is turned on so we can see more of what's going on. Start it up. -`./bin/dex-overlord --db-url=$DEX_DB_URL --key-secret=$DEX_KEY_SECRET --log-debug=true &` +`./bin/dex-overlord --db-url=$DEX_DB_URL --key-secrets=$DEX_KEY_SECRET --log-debug=true &` ## Environment Variables. diff --git a/README.md b/README.md index dda9e769..3d572115 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ A typical dex deployment consists of N dex-workers behind a load balanacer, and The dex-workers directly handle user requests, so the loss of all workers can result in service downtime. The single dex-overlord runs its tasks periodically, so it does not need to maintain 100% uptime. -## Who Should Use AuthD? +## Who Should Use Dex? **TODO** From fa96fb3a33ffcbe914dc2f9ac82da2b47683d20c Mon Sep 17 00:00:00 2001 From: Bobby Rullo Date: Wed, 26 Aug 2015 14:23:23 -0700 Subject: [PATCH 3/4] contrib: get standup up-to-date * key_secret -> key_secrets * make sure overlord starts up before other operations since it creates the db * remove race conditions by blocking on /health being up for worker and overlord --- contrib/standup-db.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/contrib/standup-db.sh b/contrib/standup-db.sh index f5bd2175..21ee8e35 100644 --- a/contrib/standup-db.sh +++ b/contrib/standup-db.sh @@ -24,6 +24,20 @@ export DEX_WORKER_DB_URL=$DEX_DB_URL # Delete/create DB dropdb $DEX_DB; createdb $DEX_DB + +DEX_KEY_SECRET=$(dd if=/dev/random bs=1 count=32 2>/dev/null | base64) + +# Start the overlord +export DEX_OVERLORD_DB_URL=$DEX_DB_URL +export DEX_OVERLORD_KEY_SECRETS=$DEX_KEY_SECRET +export DEX_OVERLORD_KEY_PERIOD=1h +./bin/dex-overlord & +echo "Waiting for overlord to start..." +until $(curl --output /dev/null --silent --fail http://localhost:5557/health); do + printf '.' + sleep 1 +done + # Create a client eval "$(./bin/dexctl -db-url=$DEX_DB_URL new-client http://127.0.0.1:5555/callback)" @@ -49,17 +63,17 @@ EOF ./bin/dexctl -db-url=$DEX_DB_URL set-connector-configs $DEX_CONNECTORS_FILE -# Start the overlord -export DEX_OVERLORD_DB_URL=$DEX_DB_URL -export DEX_OVERLORD_KEY_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -export DEX_OVERLORD_KEY_PERIOD=1h -./bin/dex-overlord & # Start the worker export DEX_WORKER_DB_URL=$DEX_DB_URL -export DEX_WORKER_KEY_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +export DEX_WORKER_KEY_SECRETS=$DEX_KEY_SECRET export DEX_WORKER_LOG_DEBUG=1 ./bin/dex-worker & +echo "Waiting for worker to start..." +until $(curl --output /dev/null --silent --fail http://localhost:5556/health); do + printf '.' + sleep 1 +done # Start the app ./bin/example-app --client-id=$DEX_APP_CLIENT_ID --client-secret=$DEX_APP_CLIENT_SECRET --discovery=http://127.0.0.1:5556 & From eb65555fe7bea8ba82cd39c8f7d32cdf0e43afc0 Mon Sep 17 00:00:00 2001 From: Bobby Rullo Date: Wed, 26 Aug 2015 14:23:49 -0700 Subject: [PATCH 4/4] build-docker-push: allow override of quay repo --- build-docker-push | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-docker-push b/build-docker-push index aa97a9ea..b90020aa 100755 --- a/build-docker-push +++ b/build-docker-push @@ -1,13 +1,15 @@ #!/bin/bash -e -registry=quay.io -repo=$registry/coreos/dex + +DOCKER_REGISTRY=${DOCKER_REGISTRY:=quay.io} +DOCKER_REPO=${DOCKER_REPO:=coreos/dex} +repo=$DOCKER_REGISTRY/$DOCKER_REPO if [ -v $DOCKER_USER ] || [ -v $DOCKER_PASSWORD ]; then echo "env variables not set: DOCKER_USER, DOCKER_PASSWORD. skipping login, assuming creds in .dockercfg" else echo logging in as $DOCKER_USER - docker login --username="$DOCKER_USER" --password="$DOCKER_PASSWORD" --email="docker.login@coreos.com" $registry + docker login --username="$DOCKER_USER" --password="$DOCKER_PASSWORD" --email="docker.login@coreos.com" $DOCKER_REGISTRY fi git_sha=$(git rev-parse HEAD)