From 2357f46a4ed2cf8b796638434e3c5bfce6b62f69 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Thu, 15 Oct 2015 17:17:31 +1100 Subject: [PATCH] CI scripts use env vars The CI scripts are configurable through the environment. They had hardcoded configurations before. The Australian BuildKite setup will need global environment variables like this: STAGING_SSH_HOST=ofn-staging2 STAGING_CURRENT_PATH=/home/openfoodweb/apps/openfoodweb/current STAGING_SERVICE=unicorn_openfoodweb STAGING_DB_HOST=localhost STAGING_DB_USER=openfoodweb STAGING_DB=openfoodweb_production PRODUCTION_REMOTE=production:/home/openfoodweb/apps/openfoodweb/current --- script/ci/includes.sh | 25 ++++++++++++++++++------ script/ci/load_staging_baseline.sh | 21 +++++++++++++------- script/ci/push_to_production.sh | 17 +++++++++------- script/ci/push_to_staging.sh | 31 ++++++++++++++++++++---------- script/ci/save_staging_baseline.sh | 9 +++++++-- 5 files changed, 71 insertions(+), 32 deletions(-) diff --git a/script/ci/includes.sh b/script/ci/includes.sh index 619d03153e..f8ccd7b4b6 100644 --- a/script/ci/includes.sh +++ b/script/ci/includes.sh @@ -5,6 +5,17 @@ function load_environment { fi } +function require_env_vars { + for var in "$@"; do + eval value=\$$var + echo "$var=$value" + if [ -z "$value" ]; then + echo "Environment variable $var missing." + exit 1 + fi + done +} + function master_merged { if [[ `git tag -l "$BUILDKITE_BRANCH"` != '' ]]; then echo "'$BUILDKITE_BRANCH' is a tag." @@ -61,14 +72,16 @@ function checkout_ofn_commit { function drop_and_recreate_database { # Adapted from: http://stackoverflow.com/questions/12924466/capistrano-with-postgresql-error-database-is-being-accessed-by-other-users - psql -U openfoodweb postgres < pg_backend_pid() -AND datname='$1'; -DROP DATABASE $1; -CREATE DATABASE $1; +AND datname='$DB'; +DROP DATABASE $DB; +CREATE DATABASE $DB; EOF } diff --git a/script/ci/load_staging_baseline.sh b/script/ci/load_staging_baseline.sh index ac7d88569f..45a57cf5c0 100755 --- a/script/ci/load_staging_baseline.sh +++ b/script/ci/load_staging_baseline.sh @@ -5,23 +5,30 @@ # current database. set -e -cd /home/openfoodweb/apps/openfoodweb/current -source ./script/ci/includes.sh +source "`dirname $0`/includes.sh" + +# We need ruby to call script/delayed_job +export PATH="$HOME/.rbenv/shims:$PATH" + +echo "Checking environment variables" +require_env_vars CURRENT_PATH SERVICE DB_HOST DB_USER DB + +cd "$CURRENT_PATH" echo "Stopping unicorn and delayed job..." -service unicorn_openfoodweb stop +service "$SERVICE" stop RAILS_ENV=staging script/delayed_job -i 0 stop echo "Backing up current data..." mkdir -p db/backup -pg_dump -h localhost -U openfoodweb openfoodweb_production |gzip > db/backup/staging-`date +%Y%m%d%H%M%S`.sql.gz +pg_dump -h "$DB_HOST" -U "$DB_USER" "$DB" |gzip > db/backup/staging-`date +%Y%m%d%H%M%S`.sql.gz echo "Loading baseline data..." -drop_and_recreate_database "openfoodweb_production" -gunzip -c db/backup/staging-baseline.sql.gz |psql -h localhost -U openfoodweb openfoodweb_production +drop_and_recreate_database "$DB" -U "$DB_USER" +gunzip -c db/backup/staging-baseline.sql.gz |psql -h "$DB_HOST" -U "$DB_USER" "$DB" echo "Restarting unicorn..." -service unicorn_openfoodweb start +service "$SERVICE" start # Delayed job is restarted by monit echo "Done!" diff --git a/script/ci/push_to_production.sh b/script/ci/push_to_production.sh index 9294bcb492..413aeeabf8 100755 --- a/script/ci/push_to_production.sh +++ b/script/ci/push_to_production.sh @@ -1,18 +1,21 @@ #!/bin/bash set -e -source ./script/ci/includes.sh +source "`dirname $0`/includes.sh" -# Add production git remote if required -PROD_TEST=`git remote | grep -s 'production' || true` -if [[ "$PROD_TEST" != *production* ]]; then - git remote add production ubuntu@ofn-prod:apps/openfoodweb/current +OFN_COMMIT=$(get_ofn_commit) +if [ "$OFN_COMMIT" = 'OFN_COMMIT_NOT_FOUND' ]; then + OFN_COMMIT=$(git rev-parse $BUILDKITE_COMMIT) fi +echo "--- Checking environment variables" +require_env_vars OFN_COMMIT STAGING_SSH_HOST STAGING_CURRENT_PATH STAGING_SERVICE STAGING_DB_HOST STAGING_DB_USER STAGING_DB PRODUCTION_REMOTE + echo "--- Saving baseline data for staging" -ssh ofn-staging2 "/home/openfoodweb/apps/openfoodweb/current/script/ci/save_staging_baseline.sh `get_ofn_commit`" +VARS="CURRENT_PATH='$STAGING_CURRENT_PATH' SERVICE='$STAGING_SERVICE' DB_HOST='$STAGING_DB_HOST' DB_USER='$STAGING_DB_USER' DB='$STAGING_DB'" +ssh "$STAGING_SSH_HOST" "$VARS $STAGING_CURRENT_PATH/script/ci/save_staging_baseline.sh $OFN_COMMIT" echo "--- Pushing to production" exec 5>&1 -OUTPUT=$(git push production `get_ofn_commit`:master --force 2>&1 |tee /dev/fd/5) +OUTPUT=$(git push "$PRODUCTION_REMOTE" "$OFN_COMMIT":master --force 2>&1 |tee /dev/fd/5) [[ $OUTPUT =~ "Done" ]] diff --git a/script/ci/push_to_staging.sh b/script/ci/push_to_staging.sh index 7ace392276..c2ba9827da 100755 --- a/script/ci/push_to_staging.sh +++ b/script/ci/push_to_staging.sh @@ -1,21 +1,32 @@ #!/bin/bash set -e -source ./script/ci/includes.sh +source "`dirname $0`/includes.sh" -# Add staging git remote if required -ST2_TEST=`git remote | grep -s 'staging2' || true` -if [[ "$ST2_TEST" != *staging2* ]]; then - git remote add staging2 openfoodweb@ofn-staging2:apps/openfoodweb/current +OFN_COMMIT=$(get_ofn_commit) +if [ "$OFN_COMMIT" = 'OFN_COMMIT_NOT_FOUND' ]; then + OFN_COMMIT=$(git rev-parse $BUILDKITE_COMMIT) +fi +STAGING_REMOTE="${STAGING_REMOTE:-$STAGING_SSH_HOST:$STAGING_CURRENT_PATH}" + +echo "--- Checking environment variables" +require_env_vars OFN_COMMIT STAGING_SSH_HOST STAGING_CURRENT_PATH STAGING_REMOTE STAGING_SERVICE STAGING_DB_HOST STAGING_DB_USER STAGING_DB + +if [ "$REQUIRE_MASTER_MERGED" = false ]; then + echo "--- NOT verifying branch is based on current master" +else + echo "--- Verifying branch is based on current master" + exit_unless_master_merged fi -echo "--- Verifying branch is based on current master" -exit_unless_master_merged - +# TODO: Optimise staging deployment +# This is stopping and re-starting unicorn and delayed job. echo "--- Loading baseline data" -ssh ofn-staging2 "/home/openfoodweb/apps/openfoodweb/current/script/ci/load_staging_baseline.sh" +VARS="CURRENT_PATH='$STAGING_CURRENT_PATH' SERVICE='$STAGING_SERVICE' DB_HOST='$STAGING_DB_HOST' DB_USER='$STAGING_DB_USER' DB='$STAGING_DB'" +ssh "$STAGING_SSH_HOST" "$VARS $STAGING_CURRENT_PATH/script/ci/load_staging_baseline.sh" +# This is stopping and re-starting unicorn and delayed job again. echo "--- Pushing to staging" exec 5>&1 -OUTPUT=$(git push staging2 `get_ofn_commit`:master --force 2>&1 |tee /dev/fd/5) +OUTPUT=$(git push "$STAGING_REMOTE" "$OFN_COMMIT":master --force 2>&1 |tee /dev/fd/5) [[ $OUTPUT =~ "Done" ]] diff --git a/script/ci/save_staging_baseline.sh b/script/ci/save_staging_baseline.sh index 9fcc6c373b..37ff60bf17 100755 --- a/script/ci/save_staging_baseline.sh +++ b/script/ci/save_staging_baseline.sh @@ -9,12 +9,17 @@ # current code checked out. set -e +source "`dirname $0`/includes.sh" -cd /home/openfoodweb/apps/openfoodweb/current +echo "Checking environment variables" +require_env_vars CURRENT_PATH SERVICE DB_HOST DB_USER DB + +cd "$CURRENT_PATH" if [[ `git rev-parse HEAD` == $1 ]]; then mkdir -p db/backup - pg_dump -h localhost -U openfoodweb openfoodweb_production |gzip > db/backup/staging-baseline.sql.gz + pg_dump -h "$DB_HOST" -U "$DB_USER" "$DB" |gzip > db/backup/staging-baseline.sql.gz echo "Staging baseline data saved." else echo "Staging SHA does not match production, we will not save staging baseline data." + echo "'`git rev-parse HEAD`' is not '$1'" fi