Cloudformation 模板创建 EMR 集群

Question

I am trying to create EMR-5.30.1 clusters with applications such as Hadoop, livy, Spark, ZooKeeper, and Hive with the help of the CloudFormation template. But the issue is with this template is I am able the cluster with only one application from the above list of applications.

below is the CloudFormation Template

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Best Practice EMR Cluster for Spark or S3 backed Hbase",
  "Parameters": {
    "EMRClusterName": {
      "Description": "Name of the cluster",
      "Type": "String",
      "Default": "emrcluster"
    },
    "KeyName": {
      "Description": "Must be an existing Keyname",
      "Type": "String",
      "Default": "keyfilename"
    },
    "MasterInstanceType": {
      "Description": "Instance type to be used for the master instance.",
      "Type": "String",
      "Default": "m5.xlarge"
    },
    "CoreInstanceType": {
      "Description": "Instance type to be used for core instances.",
      "Type": "String",
      "Default": "m5.xlarge"
    },
    "NumberOfCoreInstances": {
      "Description": "Must be a valid number",
      "Type": "Number",
      "Default": 1
    },
    "SubnetID": {
      "Description": "Must be Valid public subnet ID",
      "Default": "subnet-ee15b3e0",
      "Type": "String"
    },
    "LogUri": {
      "Description": "Must be a valid S3 URL",
      "Default": "s3://aws/elasticmapreduce/",
      "Type": "String"
    },
    "S3DataUri": {
      "Description": "Must be a valid S3 bucket URL ",
      "Default": "s3://aws/elasticmapreduce/",
      "Type": "String"
    },
    "ReleaseLabel": {
      "Description": "Must be a valid EMR release  version",
      "Default": "emr-5.30.1",
      "Type": "String"
    },
    "Applications": {
      "Description": "Please select which application will be installed on the cluster this would be either Ganglia and spark, or Ganglia and s3 backed Hbase",
      "Type": "String",
      "AllowedValues": [
        "Spark",
        "Hbase",
    "Hive",
    "Livy",
    "ZooKeeper"
    ]
     }
  },
  "Mappings": {},
  "Conditions": {
    "Spark": {
      "Fn::Equals": [
        {
          "Ref": "Applications"
        },
        "Spark"
      ]
    },
    "Hbase": {
      "Fn::Equals": [
        {
          "Ref": "Applications"
        },
        "Hbase"
      ]
    },
    "Hive": {
      "Fn::Equals": [
        {
          "Ref": "Applications"
        },
        "Hive"
      ]
    },
    "Livy": {
      "Fn::Equals": [
        {
          "Ref": "Applications"
        },
        "Livy"
      ]
    },
    "ZooKeeper": {
      "Fn::Equals": [
        {
          "Ref": "Applications"
        },
        "ZooKeeper"
      ]
    }
   },
  "Resources": {
    "EMRCluster": {
      "DependsOn": [
        "EMRClusterServiceRole",
        "EMRClusterinstanceProfileRole",
        "EMRClusterinstanceProfile"
      ],
      "Type": "AWS::EMR::Cluster",
      "Properties": {
        "Applications": [
          {
            "Name": "Ganglia"
          },
          {
            "Fn::If": [
              "Spark",
              {
                "Name": "Spark"
              },
              {
                "Ref": "AWS::NoValue"
              }
            ]
          },
          {
            "Fn::If": [
              "Hbase",
              {
                "Name": "Hbase"
              },
              {
                "Ref": "AWS::NoValue"
              }
            ]
          },
      {
            "Fn::If": [
              "Hive",
              {
                "Name": "Hive"
              },
              {
                "Ref": "AWS::NoValue"
              }
            ]
          },
      {
            "Fn::If": [
              "Livy",
              {
                "Name": "Livy"
              },
              {
                "Ref": "AWS::NoValue"
              }
            ]
          },
          {
            "Fn::If": [
              "ZooKeeper",
              {
                "Name": "ZooKeeper"
              },
              {
                "Ref": "AWS::NoValue"
              }
            ]
          }
    ],
        "Configurations": [
          {
            "Classification": "hbase-site",
            "ConfigurationProperties": {
              "hbase.rootdir":{"Ref":"S3DataUri"}
            }
          },
          {
            "Classification": "hbase",
            "ConfigurationProperties": {
              "hbase.emr.storageMode": "s3"
            }
          }
        ],
        "Instances": {
          "Ec2KeyName": {
            "Ref": "KeyName"
          },
          "Ec2SubnetId": {
            "Ref": "SubnetID"
          },
          "MasterInstanceGroup": {
            "InstanceCount": 1,
            "InstanceType": {
              "Ref": "MasterInstanceType"
            },
            "Market": "ON_DEMAND",
            "Name": "Master"
          },
          "CoreInstanceGroup": {
            "InstanceCount": {
              "Ref": "NumberOfCoreInstances"
            },
            "InstanceType": {
              "Ref": "CoreInstanceType"
            },
            "Market": "ON_DEMAND",
            "Name": "Core"
          },
          "TerminationProtected": false
        },
        "VisibleToAllUsers": true,
        "JobFlowRole": {
          "Ref": "EMRClusterinstanceProfile"
        },
        "ReleaseLabel": {
          "Ref": "ReleaseLabel"
        },
        "LogUri": {
          "Ref": "LogUri"
        },
        "Name": {
          "Ref": "EMRClusterName"
        },
        "AutoScalingRole": "EMR_AutoScaling_DefaultRole",
        "ServiceRole": {
          "Ref": "EMRClusterServiceRole"
        }
      }
    },
    "EMRClusterServiceRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  "elasticmapreduce.amazonaws.com"
                ]
              },
              "Action": [
                "sts:AssumeRole"
              ]
            }
          ]
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
        ],
        "Path": "/"
      }
    },
    "EMRClusterinstanceProfileRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  "ec2.amazonaws.com"
                ]
              },
              "Action": [
                "sts:AssumeRole"
              ]
            }
          ]
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
        ],
        "Path": "/"
      }
    },
    "EMRClusterinstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Path": "/",
        "Roles": [
          {
            "Ref": "EMRClusterinstanceProfileRole"
          }
        ]
      }
    }
  },
  "Outputs": {}
}

Also, I want to add a bootstrap script in this template as well, Can anyone please help me with the issue.

Answer 1

As per my knoweldge and understanding, Applications in your case should be an array like below, as mentioned in documentation

 "Applications" : [ Application, ... ],

In you case, you can list applications like

 "Applications" : [ 
     {"Name" : "Spark"},
{"Name" : "Hbase"},
{"Name" : "Hive"},
{"Name" : "Livy"},
{"Name" : "Zookeeper"},
]

For more arguments other than Name to individual application dictionary , see detail here , you can pass Args , Additional_info etc

Answer 2

You can use following way:-

If you set "ReleaseLabel" then there is no need to mention versions of applications

"Applications": [{ "Name": "Hive" }, { "Name": "Presto" }, { "Name": "Spark" } ]

For bootstrap:-

"BootstrapActions": [{ "Name": "setup", "ScriptBootstrapAction": { "Path": "s3://bucket/key/Bootstrap.sh" } }]

Answer 3

Define like this to create all applications at once.

{
    "Type": "AWS::EMR::Cluster",
    "Properties": {
        "Applications": [
            {
                "Name": "Ganglia"
            },
            {
                "Name": "Spark"
            },
            {
                "Name": "Livy"
            },
            {
                "Name": "ZooKeeper"
            },
            {
                "Name": "JupyterHub"
            }
        ]
    }
}