From 0394ea729edbd207d541bf39f94a73c120c79af6 Mon Sep 17 00:00:00 2001 From: Space Date: Fri, 10 Apr 2015 13:28:23 -0700 Subject: [PATCH] Broke up string to make script more readable in template.cfn --- labs/kafka/aws/template.cfn | 68 ++++++++++++++++++- .../aws/user-data-script-for-cfn-template.sh | 2 +- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/labs/kafka/aws/template.cfn b/labs/kafka/aws/template.cfn index 2bafd92..f067d9a 100755 --- a/labs/kafka/aws/template.cfn +++ b/labs/kafka/aws/template.cfn @@ -133,7 +133,10 @@ "Fn::Join": [ "", [ - "#!/bin/bash -ex\nexec > >(tee /var/log/user-data.log) 2>&1\n\necho BEGIN\n", + "#!/bin/bash -ex\n", + "exec &> >(tee /var/log/user-data.log)\n", + "\n", + "echo BEGIN\n", "\n", "REGION=\"", { "Ref": "AWS::Region" }, "\"\n", "DEPLOYMENT=\"", { "Ref": "Deployment" }, "\"\n", @@ -142,7 +145,68 @@ "ZK_VERSION=\"", { "Ref": "ZookeeperVersion" }, "\"\n", "INSTANCE_WAIT_HANDLE_URL=\"", { "Ref": "WaitForInstanceWaitHandle" }, "\"\n", "\n", - "WORKING_DIR=\"/deploy\"\nREPO_DIR=\"$WORKING_DIR/repo\"\nLAB_PATH=\"labs/kafka\"\nINSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\nRUBY_URL=\"https://rvm_io.global.ssl.fastly.net/binaries/ubuntu/14.04/x86_64/ruby-2.1.5.tar.bz2\"\n\n# Update repos and install dependencies\napt-get update\napt-get -y install git-core build-essential awscli\n\n# Install rvm for the latest ruby version\ncommand curl -sSL https://rvm.io/mpapis.asc | gpg --import -\ncurl -sSL https://get.rvm.io | bash -s stable\nsource /usr/local/rvm/scripts/rvm\necho \"$RUBY_URL=1a201d082586036092cfc5b79dd26718\" >> /usr/local/rvm/user/md5\necho \"$RUBY_URL=91216074cb5f66ef5e33d47e5d3410148cc672dc73cc0d9edff92e00d20c9973bec7ab21a3462ff4e9ff9b23eff952e83b51b96a3b11cb5c23be587046eb0c57\" >> /usr/local/rvm/user/sha512\nrvm mount -r $RUBY_URL --verify-downloads 1\nrvm use 2.1 --default\nrvm rubygems current\n\n# Get latest version of jq\nwget https://stedolan.github.io/jq/download/linux64/jq -O /usr/local/bin/jq\nchmod +x /usr/local/bin/jq\n\ngit clone https://git@github.com/stealthly/minotaur.git \"$REPO_DIR\"\n\n# Install Chef\ncurl -L https://www.opscode.com/chef/install.sh | bash\n\n# Install Bundler and community cookbooks with librarian\naws s3 cp --region $REGION s3://bdoss-deploy/gems/librarian-0.1.2.gem /tmp/librarian-0.1.2.gem\ngem install /tmp/librarian-0.1.2.gem --no-ri --no-rdoc\ngem install bundler --no-ri --no-rdoc\ncd $REPO_DIR/$LAB_PATH/chef/ && bundle install && librarian-chef install\n\n# Find kafka nodes that belong to the same deployment and environment\nNODES_FILTER=\"Name=tag:Name,Values=kafka.$DEPLOYMENT.$ENVIRONMENT\"\nQUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\nKAFKA_BROKERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n\n# Find zookeeper nodes that belong to the same deployment and environment\nNODES_FILTER=\"Name=tag:Name,Values=zookeeper.$DEPLOYMENT.$ENVIRONMENT\"\nQUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\nZK_SERVERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n\n# Run Chef\nkafka_url=\"$KAFKA_URL\" \\\nzk_version=\"$ZK_VERSION\" \\\nzk_servers=\"$ZK_SERVERS\" \\\nkafka_brokers=$\"$KAFKA_BROKERS\" \\\nchef-solo -c \"$REPO_DIR/$LAB_PATH/chef/solo.rb\" -j \"$REPO_DIR/$LAB_PATH/chef/solo_kb.json\"\n\n# Create default topics, such as \"dataset\" and \"mirror_dataset\"\nsleep 15\n/opt/apache/kafka/bin/kafka-topics.sh --create --topic dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n/opt/apache/kafka/bin/kafka-topics.sh --create --topic mirror_dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n\n# Notify wait handle\nWAIT_HANDLE_JSON=\"{\\\"Status\\\": \\\"SUCCESS\\\", \\\"Reason\\\": \\\"Done\\\", \\\"UniqueId\\\": \\\"1\\\", \\\"Data\\\": \\\"$INSTANCE_ID\\\"}\"\ncurl -X PUT -H 'Content-Type:' --data-binary \"$WAIT_HANDLE_JSON\" \"$INSTANCE_WAIT_HANDLE_URL\"\n\necho END\n" + "WORKING_DIR=\"/deploy\"\n", + "REPO_DIR=\"$WORKING_DIR/repo\"\n", + "LAB_PATH=\"labs/kafka\"\n", + "INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)\n", + "RUBY_URL=\"https://rvm_io.global.ssl.fastly.net/binaries/ubuntu/14.04/x86_64/ruby-2.1.5.tar.bz2\"\n", + "\n", + "# Update repos and install dependencies\n", + "apt-get update\n", + "apt-get -y install git-core build-essential awscli\n", + "\n", + "# Install rvm for the latest ruby version\n", + "command curl -sSL https://rvm.io/mpapis.asc | gpg --import -\n", + "curl -sSL https://get.rvm.io | bash -s stable\n", + "source /usr/local/rvm/scripts/rvm\n", + "echo \"$RUBY_URL=1a201d082586036092cfc5b79dd26718\" >> /usr/local/rvm/user/md5\n", + "echo \"$RUBY_URL=91216074cb5f66ef5e33d47e5d3410148cc672dc73cc0d9edff92e00d20c9973bec7ab21a3462ff4e9ff9b23eff952e83b51b96a3b11cb5c23be587046eb0c57\" >> /usr/local/rvm/user/sha512\n", + "rvm mount -r $RUBY_URL --verify-downloads 1\n", + "rvm use 2.1 --default\n", + "rvm rubygems current\n", + "\n", + "# Get latest version of jq\n", + "wget https://stedolan.github.io/jq/download/linux64/jq -O /usr/local/bin/jq\n", + "chmod +x /usr/local/bin/jq\n", + "\n", + "git clone https://git@github.com/stealthly/minotaur.git \"$REPO_DIR\"\n", + "\n", + "# Install Chef\n", + "curl -L https://www.opscode.com/chef/install.sh | bash\n", + "\n", + "# Install Bundler and community cookbooks with librarian\n", + "aws s3 cp --region $REGION s3://bdoss-deploy/gems/librarian-0.1.2.gem /tmp/librarian-0.1.2.gem\n", + "gem install /tmp/librarian-0.1.2.gem --no-ri --no-rdoc\n", + "gem install bundler --no-ri --no-rdoc\n", + "cd $REPO_DIR/$LAB_PATH/chef/ && bundle install && librarian-chef install\n", + "\n", + "# Find kafka nodes that belong to the same deployment and environment\n", + "NODES_FILTER=\"Name=tag:Name,Values=kafka.$DEPLOYMENT.$ENVIRONMENT\"\n", + "QUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\n", + "KAFKA_BROKERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n", + "\n", + "# Find zookeeper nodes that belong to the same deployment and environment\n", + "NODES_FILTER=\"Name=tag:Name,Values=zookeeper.$DEPLOYMENT.$ENVIRONMENT\"\n", + "QUERY=\"Reservations[].Instances[].NetworkInterfaces[].PrivateIpAddress\"\n", + "ZK_SERVERS=$(aws ec2 describe-instances --region \"$REGION\" --filters \"$NODES_FILTER\" --query \"$QUERY\" | jq --raw-output 'join(\",\")')\n", + "\n", + "# Run Chef\n", + "kafka_url=\"$KAFKA_URL\" \\\n", + "zk_version=\"$ZK_VERSION\" \\\n", + "zk_servers=\"$ZK_SERVERS\" \\\n", + "kafka_brokers=$\"$KAFKA_BROKERS\" \\\n", + "chef-solo -c \"$REPO_DIR/$LAB_PATH/chef/solo.rb\" -j \"$REPO_DIR/$LAB_PATH/chef/solo_kb.json\"\n", + "\n", + "# Create default topics, such as \"dataset\" and \"mirror_dataset\"\n", + "sleep 15\n", + "/opt/apache/kafka/bin/kafka-topics.sh --create --topic dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n", + "/opt/apache/kafka/bin/kafka-topics.sh --create --topic mirror_dataset --partitions 1 --replication-factor 1 --zookeeper $(expr $ZK_SERVERS : '\\([0-9\\.]*\\)')\n", + "\n", + "# Notify wait handle\n", + "WAIT_HANDLE_JSON=\"{\\\"Status\\\": \\\"SUCCESS\\\", \\\"Reason\\\": \\\"Done\\\", \\\"UniqueId\\\": \\\"1\\\", \\\"Data\\\": \\\"$INSTANCE_ID\\\"}\"\n", + "curl -X PUT -H 'Content-Type:' --data-binary \"$WAIT_HANDLE_JSON\" \"$INSTANCE_WAIT_HANDLE_URL\"\n", + "\n", + "echo END\n" ] ] } diff --git a/labs/kafka/aws/user-data-script-for-cfn-template.sh b/labs/kafka/aws/user-data-script-for-cfn-template.sh index 27465a9..d257878 100644 --- a/labs/kafka/aws/user-data-script-for-cfn-template.sh +++ b/labs/kafka/aws/user-data-script-for-cfn-template.sh @@ -19,7 +19,7 @@ # This script is meant to be JSON escaped and pasted into an AWS CloudFormation # instance's UserData property. -exec > >(tee /var/log/user-data.log) 2>&1 +exec &> >(tee /var/log/user-data.log) echo BEGIN