From c89520993657597492e95645462ee596a8b457e2 Mon Sep 17 00:00:00 2001 From: Steve Harms Date: Mon, 25 Apr 2016 09:32:19 -0400 Subject: [PATCH 1/6] Initial script to create additional spaces --- create-space.sh | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 create-space.sh diff --git a/create-space.sh b/create-space.sh new file mode 100755 index 00000000..6508bdfd --- /dev/null +++ b/create-space.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +if [ ! -n "$1" ]; then + echo "Usage: create-space.sh spacename" + exit 1 +fi +echo "Creating space: $1" +SPACE=$1 + +# Creating the indexing space +cf create-space ${SPACE} -o ed + +echo "*** When creating the space, the user which creates it is a SpaceManager" +echo "*** Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" + +# Target the space +cf target -o ed -s ${SPACE} + +# Create a bservice instance used by snapshot tools +cf create-service s3 basic data-files +#cf create-user-provided-service bservice -p '{"BSERVICE_ACCESS_KEY":"YOUR_S3_ACCESS_KEY", "BSERVICE_SECRET_KEY": "YOUR_S3_SECRET_KEY", "BSERVICE_BUCKET": "YOUR_S3_BUCKET"}' + +# Create backup service +echo "To create a backup service in this space run:" +echo "cf create-service s3 basic backup" + +# Create the ElasticSearch service +cf create-service elasticsearch-swarm-1.7.5 3x eservice + +# Create ccapi-indexing +#cf push -f ccapi-${SPACE} + +# /downloads happens through an app in production space +# ed-public-download started 2/2 64M 1G ed-public-download.apps.cloud.gov, download.collegescorecard.ed.gov +# connected to bucket .... + From 1d8b0cbe362d1572a4643230ed8cba5cc2ca7eae Mon Sep 17 00:00:00 2001 From: Steve Harms Date: Mon, 25 Apr 2016 09:50:54 -0400 Subject: [PATCH 2/6] Clear up documentation and use hashes to assist in cut and paste --- create-space.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/create-space.sh b/create-space.sh index 6508bdfd..01dd95b4 100755 --- a/create-space.sh +++ b/create-space.sh @@ -11,27 +11,38 @@ SPACE=$1 # Creating the indexing space cf create-space ${SPACE} -o ed -echo "*** When creating the space, the user which creates it is a SpaceManager" -echo "*** Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" +echo "# When creating the space, the user which creates it is a SpaceManager" +echo "# Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" # Target the space cf target -o ed -s ${SPACE} # Create a bservice instance used by snapshot tools cf create-service s3 basic data-files + + #cf create-user-provided-service bservice -p '{"BSERVICE_ACCESS_KEY":"YOUR_S3_ACCESS_KEY", "BSERVICE_SECRET_KEY": "YOUR_S3_SECRET_KEY", "BSERVICE_BUCKET": "YOUR_S3_BUCKET"}' # Create backup service -echo "To create a backup service in this space run:" -echo "cf create-service s3 basic backup" +echo "# To create a backup service in this space run:" +echo "# cf create-service s3 basic backup" # Create the ElasticSearch service cf create-service elasticsearch-swarm-1.7.5 3x eservice -# Create ccapi-indexing -#cf push -f ccapi-${SPACE} - -# /downloads happens through an app in production space -# ed-public-download started 2/2 64M 1G ed-public-download.apps.cloud.gov, download.collegescorecard.ed.gov -# connected to bucket .... +echo "# To create the API server push the ccapi-${SPACE} app:" +echo "cf push -f manifest-${SPACE}.yml" + +echo "# For data archive / downloads, these are served via a S3 proxy" +echo "# The /downloads path is redirected via CloudFront to" +echo "# ed-public-download.apps.cloud.gov which is in the production space" +echo "# To create additional S3 proxies: https://github.com/18F/cg-s3-proxy" + +echo "# now you need to index" +echo "cf-ssh -f manifest-${SPACE}" +echo "# wait several minutes for this to connect" +echo "echo $DATA_PATH" +echo "# if blank, you will get default cities data" +echo "rake import" +echo "# when this is done, go to https://ccapi-${SPACE}.18f.gov and explore" From f923eba1f452a60a7fe6a9b06d2917ea7181e7af Mon Sep 17 00:00:00 2001 From: Sarah Allen Date: Mon, 25 Apr 2016 07:07:45 -0700 Subject: [PATCH 3/6] =?UTF-8?q?code=20changes=20to=20go=20with=20doc?= =?UTF-8?q?=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit not sure if we should keep manifest-testing.yml --- create-space.sh | 13 ++++++++----- lib/data_magic.rb | 7 +++++-- manifest-testing.yml | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 manifest-testing.yml diff --git a/create-space.sh b/create-space.sh index 01dd95b4..cb041d54 100755 --- a/create-space.sh +++ b/create-space.sh @@ -2,7 +2,8 @@ set -e if [ ! -n "$1" ]; then - echo "Usage: create-space.sh spacename" + echo "Usage: From inside open-data-maker directory..." + echo " create-space.sh spacename" exit 1 fi echo "Creating space: $1" @@ -12,7 +13,7 @@ SPACE=$1 cf create-space ${SPACE} -o ed echo "# When creating the space, the user which creates it is a SpaceManager" -echo "# Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" +echo "# Add developers using: cf set-space-role USERNAME ed ${SPACE} SpaceDeveloper" # Target the space cf target -o ed -s ${SPACE} @@ -30,14 +31,17 @@ echo "# cf create-service s3 basic backup" # Create the ElasticSearch service cf create-service elasticsearch-swarm-1.7.5 3x eservice -echo "# To create the API server push the ccapi-${SPACE} app:" -echo "cf push -f manifest-${SPACE}.yml" +echo "Creating the API server by pushing the ccapi-${SPACE} app:" +cf push -f manifest-${SPACE}.yml +echo "By default the app will use the data-files bucket, leaving DATA_PATH env blank" echo "# For data archive / downloads, these are served via a S3 proxy" echo "# The /downloads path is redirected via CloudFront to" echo "# ed-public-download.apps.cloud.gov which is in the production space" echo "# To create additional S3 proxies: https://github.com/18F/cg-s3-proxy" +echo "TODO: how to put files in the bucket" + echo "# now you need to index" echo "cf-ssh -f manifest-${SPACE}" echo "# wait several minutes for this to connect" @@ -45,4 +49,3 @@ echo "echo $DATA_PATH" echo "# if blank, you will get default cities data" echo "rake import" echo "# when this is done, go to https://ccapi-${SPACE}.18f.gov and explore" - diff --git a/lib/data_magic.rb b/lib/data_magic.rb index 1000d5cf..e3ff6eaa 100644 --- a/lib/data_magic.rb +++ b/lib/data_magic.rb @@ -41,8 +41,11 @@ class InvalidDictionary < StandardError def self.s3 if @s3.nil? - if ENV['VCAP_APPLICATION'] - s3cred = ::CF::App::Credentials.find_by_service_name(ENV['s3_bucket_service'] || 'bservice') + if ENV['VCAP_APPLICATION'] # in Cloud Foundry + cfcred = ::CF::App::Credentials.find_by_service_name(ENV['s3_bucket_service'] || 'data-files') + s3cred = {'access_key'=> cfcred['access_key_id'], 'secret_key' => cfcred['secret_access_key']} + bucket_name = cfcred['bucket'] + // TODO: default DATA_PATH to s3://{bucket_name} else s3cred = {'access_key'=> ENV['s3_access_key'], 'secret_key' => ENV['s3_secret_key']} end diff --git a/manifest-testing.yml b/manifest-testing.yml new file mode 100644 index 00000000..94f74014 --- /dev/null +++ b/manifest-testing.yml @@ -0,0 +1,16 @@ +--- +applications: +- name: ccapi + host: ccapi-testing + domain: 18f.gov + command: bundle exec puma -C ./config/puma.rb + instances: 1 + memory: 1G + services: + - data-files + - eservice + env: + MAX_THREADS: 5 + WEB_CONCURRENCY: 1 + INDEX_APP: enable + NPROCS: 1 From 72b362bad546975da458632b36d85602f1761adc Mon Sep 17 00:00:00 2001 From: Jacob Harris Date: Fri, 29 Apr 2016 15:05:55 -0400 Subject: [PATCH 4/6] create-space works with default data Error in the a comment character was blowing up cf push --- create-space.sh | 4 ++-- lib/data_magic.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/create-space.sh b/create-space.sh index cb041d54..ea3c9a66 100755 --- a/create-space.sh +++ b/create-space.sh @@ -43,9 +43,9 @@ echo "# To create additional S3 proxies: https://github.com/18F/cg-s3-proxy" echo "TODO: how to put files in the bucket" echo "# now you need to index" -echo "cf-ssh -f manifest-${SPACE}" +echo "cf-ssh -f manifest-${SPACE}.yml --verbose" echo "# wait several minutes for this to connect" echo "echo $DATA_PATH" -echo "# if blank, you will get default cities data" +echo "# it should be blank, meaning you will get default cities data" echo "rake import" echo "# when this is done, go to https://ccapi-${SPACE}.18f.gov and explore" diff --git a/lib/data_magic.rb b/lib/data_magic.rb index e3ff6eaa..b48879ca 100644 --- a/lib/data_magic.rb +++ b/lib/data_magic.rb @@ -45,7 +45,7 @@ def self.s3 cfcred = ::CF::App::Credentials.find_by_service_name(ENV['s3_bucket_service'] || 'data-files') s3cred = {'access_key'=> cfcred['access_key_id'], 'secret_key' => cfcred['secret_access_key']} bucket_name = cfcred['bucket'] - // TODO: default DATA_PATH to s3://{bucket_name} + # TODO: default DATA_PATH to s3://{bucket_name} else s3cred = {'access_key'=> ENV['s3_access_key'], 'secret_key' => ENV['s3_secret_key']} end From 4f27f34729b6324f04bfdf5d1604cf93cd7fb753 Mon Sep 17 00:00:00 2001 From: Jacob Harris Date: Tue, 3 May 2016 12:52:35 -0400 Subject: [PATCH 5/6] Add symlink .cfignore -> .gitignore --- .cfignore | 1 + .gitignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 .cfignore diff --git a/.cfignore b/.cfignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.cfignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index a395c345..ca37456c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ db/*.db .*.sw* .env .*.env -.cfignore .vagrant .idea/ *profile* From a3a1baaa38223018ea5606d14a14e803f0d5c6b3 Mon Sep 17 00:00:00 2001 From: Jacob Harris Date: Wed, 18 May 2016 13:51:18 -0400 Subject: [PATCH 6/6] Updated scripts to work with CloudFoundry --- script/s3config.rb | 35 ++++++++++++++++++++++------------- script/s3pull | 2 +- script/s3push | 2 +- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/script/s3config.rb b/script/s3config.rb index d54fb6c3..c0d993dd 100644 --- a/script/s3config.rb +++ b/script/s3config.rb @@ -4,6 +4,7 @@ # @s3 = ::Aws::S3::Client.new require 'dotenv' +require 'json' branch = `echo $(git symbolic-ref --short HEAD)`.chomp @@ -12,26 +13,34 @@ puts "using APP_ENV from environment #{APP_ENV}" else case branch - when "master" - APP_ENV = "production" - when "staging" - APP_ENV = "staging" - else - puts "not on master or staging branch lets use dev" - APP_ENV = "dev" + when "master" + APP_ENV = "production" + when "staging" + APP_ENV = "staging" + else + puts "not on master or staging branch lets use dev" + APP_ENV = "dev" # FIXME: shouldn't the APP_ENV be testing? end end -Dotenv.load( - File.expand_path("../../.#{APP_ENV}.env", __FILE__), - File.expand_path("../../.env", __FILE__)) +cf_credentials = `cf target -o ed -s #{APP_ENV} && echo "$(cf env ccapi)" | tail -n +5 | sed -n -e :a -e '1,10!{P;N;D;};N;ba'` +cf_json_str = cf_credentials.gsub("\n", '').gsub(/^[^{]+{/, '{').gsub(/{\s+"VCAP_APPLICATION".+$/, '') + +cf_json = JSON.parse(cf_json_str) +cf_data_files = cf_json['VCAP_SERVICES']['s3'].detect {|j| j['name'] == 'data-files'} + +fail "Unable to find data-files configuration" if cf_data_files.nil? + +ENV['CF_CREDENTIALS'] = cf_json_str +ENV['AWS_ACCESS_KEY_ID'] = cf_data_files['credentials']['access_key_id'] +ENV['AWS_SECRET_ACCESS_KEY'] = cf_data_files['credentials']['secret_access_key'] +ENV['BUCKET_NAME'] = cf_data_files['credentials']['bucket'] require 'aws-sdk' puts "app env: #{APP_ENV}" -puts "bucket name: #{ENV['s3_bucket']}" - +puts "bucket name: #{ENV['BUCKET_NAME']}" -s3cred = {'access_key'=> ENV['s3_access_key'], 'secret_key' => ENV['s3_secret_key']} +s3cred = {'access_key'=> ENV['AWS_ACCESS_KEY_ID'], 'secret_key' => ENV['AWS_SECRET_ACCESS_KEY']} ::Aws.config[:credentials] = ::Aws::Credentials.new(s3cred['access_key'], s3cred['secret_key']) ::Aws.config[:region] = 'us-east-1' diff --git a/script/s3pull b/script/s3pull index 7cb48ec3..e9647f9c 100755 --- a/script/s3pull +++ b/script/s3pull @@ -4,7 +4,7 @@ require_relative 's3config.rb' @s3 = ::Aws::S3::Client.new -bucket = ENV['s3_bucket'] +bucket = ENV['BUCKET_NAME'] dirname = 'real-data' unless File.directory?(dirname) diff --git a/script/s3push b/script/s3push index 6d300280..c4ed1bee 100755 --- a/script/s3push +++ b/script/s3push @@ -5,7 +5,7 @@ require_relative 's3config.rb' @s3 = ::Aws::S3::Client.new dirname = 'real-data' -bucket_name = ENV['s3_bucket'] +bucket_name = ENV['BUCKET_NAME'] datayamlpath = File.expand_path("../../#{dirname}/#{bucket_name}.yaml", __FILE__) puts "copying #{datayamlpath}"