Skip to content

Broke up string to make script more readable in template.cfn #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kecaps wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Broke up string to make script more readable in template.cfn #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 66 additions & 2 deletions labs/kafka/aws/template.cfn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,10 @@
"Fn::Join": [
"",
[
"#!/bin/bash -ex\nexec > >(tee /var/log/user-data.log) 2>&1\n\necho BEGIN\n",
"#!/bin/bash -ex\n",
"exec &> >(tee /var/log/user-data.log)\n",
"\n",
"echo BEGIN\n",
"\n",
"REGION=\"", { "Ref": "AWS::Region" }, "\"\n",
"DEPLOYMENT=\"", { "Ref": "Deployment" }, "\"\n",
Expand All @@ -142,7 +145,68 @@
"ZK_VERSION=\"", { "Ref": "ZookeeperVersion" }, "\"\n",
"INSTANCE_WAIT_HANDLE_URL=\"", { "Ref": "WaitForInstanceWaitHandle" }, "\"\n",
"\n",
"WORKING_DIR=\"/deploy\"\nREPO_DIR=\"$WORKING_DIR/repo\"\nLAB_PATH=\"labs/kafka\"\nINSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\nRUBY_URL=\"https://rvm_io.global.ssl.fastly.net/binaries/ubuntu/14.04/x86_64/ruby-2.1.5.tar.bz2\"\n\n# Update repos and install dependencies\napt-get update\napt-get -y install git-core build-essential awscli\n\n# Install rvm for the latest ruby version\ncommand curl -sSL https://rvm.io/mpapis.asc | gpg --import -\ncurl -sSL https://get.rvm.io | bash -s stable\nsource /usr/local/rvm/scripts/rvm\necho \"$RUBY_URL=1a201d082586036092cfc5b79dd26718\" >> /usr/local/rvm/user/md5\necho \"$RUBY_URL=91216074cb5f66ef5e33d47e5d3410148cc672dc73cc0d9edff92e00d20c9973bec7ab21a3462ff4e9ff9b23eff952e83b51b96a3b11cb5c23be587046eb0c57\" >> /usr/local/rvm/user/sha512\nrvm mount -r $RUBY_URL --verify-downloads 1\nrvm use 2.1 --default\nrvm rubygems current\n\n# Get latest version of jq\nwget https://stedolan.github.io/jq/download/linux64/jq -O /usr/local/bin/jq\nchmod +x /usr/local/bin/jq\n\ngit clone https://git@github.com/stealthly/minotaur.git \"$REPO_DIR\"\n\n# Install Chef\ncurl -L https://www.opscode.com/chef/install.sh | bash\n\n# Install Bundler and community cookbooks with librarian\naws s3 cp --region $REGION s3://bdoss-deploy/gems/librarian-0.1.2.gem /tmp/librarian-0.1.2.gem\ngem install /tmp/librarian-0.1.2.gem --no-ri --no-rdoc\ngem install bundler --no-ri --no-rdoc\ncd $REPO_DIR/$LAB_PATH/chef/ && bundle install && librarian-chef install\n\n# Find kafka nodes that belong to the same deployment and environment\nNODES_FILTER=\"Name=tag:Name,Values=kafka.$DEPLOYMENT.$ENVIRONMENT\"\nQUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\nKAFKA_BROKERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n\n# Find zookeeper nodes that belong to the same deployment and environment\nNODES_FILTER=\"Name=tag:Name,Values=zookeeper.$DEPLOYMENT.$ENVIRONMENT\"\nQUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\nZK_SERVERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n\n# Run Chef\nkafka_url=\"$KAFKA_URL\" \\\nzk_version=\"$ZK_VERSION\" \\\nzk_servers=\"$ZK_SERVERS\" \\\nkafka_brokers=$\"$KAFKA_BROKERS\" \\\nchef-solo -c \"$REPO_DIR/$LAB_PATH/chef/solo.rb\" -j \"$REPO_DIR/$LAB_PATH/chef/solo_kb.json\"\n\n# Create default topics, such as \"dataset\" and \"mirror_dataset\"\nsleep 15\n/opt/apache/kafka/bin/kafka-topics.sh --create --topic dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n/opt/apache/kafka/bin/kafka-topics.sh --create --topic mirror_dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n\n# Notify wait handle\nWAIT_HANDLE_JSON=\"{\\\"Status\\\": \\\"SUCCESS\\\", \\\"Reason\\\": \\\"Done\\\", \\\"UniqueId\\\": \\\"1\\\", \\\"Data\\\": \\\"$INSTANCE_ID\\\"}\"\ncurl -X PUT -H 'Content-Type:' --data-binary \"$WAIT_HANDLE_JSON\" \"$INSTANCE_WAIT_HANDLE_URL\"\n\necho END\n"
"WORKING_DIR=\"/deploy\"\n",
"REPO_DIR=\"$WORKING_DIR/repo\"\n",
"LAB_PATH=\"labs/kafka\"\n",
"INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\n",
"RUBY_URL=\"https://rvm_io.global.ssl.fastly.net/binaries/ubuntu/14.04/x86_64/ruby-2.1.5.tar.bz2\"\n",
"\n",
"# Update repos and install dependencies\n",
"apt-get update\n",
"apt-get -y install git-core build-essential awscli\n",
"\n",
"# Install rvm for the latest ruby version\n",
"command curl -sSL https://rvm.io/mpapis.asc | gpg --import -\n",
"curl -sSL https://get.rvm.io | bash -s stable\n",
"source /usr/local/rvm/scripts/rvm\n",
"echo \"$RUBY_URL=1a201d082586036092cfc5b79dd26718\" >> /usr/local/rvm/user/md5\n",
"echo \"$RUBY_URL=91216074cb5f66ef5e33d47e5d3410148cc672dc73cc0d9edff92e00d20c9973bec7ab21a3462ff4e9ff9b23eff952e83b51b96a3b11cb5c23be587046eb0c57\" >> /usr/local/rvm/user/sha512\n",
"rvm mount -r $RUBY_URL --verify-downloads 1\n",
"rvm use 2.1 --default\n",
"rvm rubygems current\n",
"\n",
"# Get latest version of jq\n",
"wget https://stedolan.github.io/jq/download/linux64/jq -O /usr/local/bin/jq\n",
"chmod +x /usr/local/bin/jq\n",
"\n",
"git clone https://git@github.com/stealthly/minotaur.git \"$REPO_DIR\"\n",
"\n",
"# Install Chef\n",
"curl -L https://www.opscode.com/chef/install.sh | bash\n",
"\n",
"# Install Bundler and community cookbooks with librarian\n",
"aws s3 cp --region $REGION s3://bdoss-deploy/gems/librarian-0.1.2.gem /tmp/librarian-0.1.2.gem\n",
"gem install /tmp/librarian-0.1.2.gem --no-ri --no-rdoc\n",
"gem install bundler --no-ri --no-rdoc\n",
"cd $REPO_DIR/$LAB_PATH/chef/ && bundle install && librarian-chef install\n",
"\n",
"# Find kafka nodes that belong to the same deployment and environment\n",
"NODES_FILTER=\"Name=tag:Name,Values=kafka.$DEPLOYMENT.$ENVIRONMENT\"\n",
"QUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\n",
"KAFKA_BROKERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n",
"\n",
"# Find zookeeper nodes that belong to the same deployment and environment\n",
"NODES_FILTER=\"Name=tag:Name,Values=zookeeper.$DEPLOYMENT.$ENVIRONMENT\"\n",
"QUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\n",
"ZK_SERVERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n",
"\n",
"# Run Chef\n",
"kafka_url=\"$KAFKA_URL\" \\\n",
"zk_version=\"$ZK_VERSION\" \\\n",
"zk_servers=\"$ZK_SERVERS\" \\\n",
"kafka_brokers=$\"$KAFKA_BROKERS\" \\\n",
"chef-solo -c \"$REPO_DIR/$LAB_PATH/chef/solo.rb\" -j \"$REPO_DIR/$LAB_PATH/chef/solo_kb.json\"\n",
"\n",
"# Create default topics, such as \"dataset\" and \"mirror_dataset\"\n",
"sleep 15\n",
"/opt/apache/kafka/bin/kafka-topics.sh --create --topic dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n",
"/opt/apache/kafka/bin/kafka-topics.sh --create --topic mirror_dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n",
"\n",
"# Notify wait handle\n",
"WAIT_HANDLE_JSON=\"{\\\"Status\\\": \\\"SUCCESS\\\", \\\"Reason\\\": \\\"Done\\\", \\\"UniqueId\\\": \\\"1\\\", \\\"Data\\\": \\\"$INSTANCE_ID\\\"}\"\n",
"curl -X PUT -H 'Content-Type:' --data-binary \"$WAIT_HANDLE_JSON\" \"$INSTANCE_WAIT_HANDLE_URL\"\n",
"\n",
"echo END\n"
]
]
}
Expand Down
2 changes: 1 addition & 1 deletion labs/kafka/aws/user-data-script-for-cfn-template.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# This script is meant to be JSON escaped and pasted into an AWS CloudFormation
# instance's UserData property.

exec > >(tee /var/log/user-data.log) 2>&1
exec &> >(tee /var/log/user-data.log)

echo BEGIN

Expand Down