chef/data_bags/crowbar/bc-template-hadoop.json
{
"id": "bc-template-hadoop",
"description": "Common libraries and utilities that provides the basic Hadoop runtime environment (HDFS/Map Reduce). A set of components and interfaces which implements a distributed filesystem and provides general I/O access for the hadoop framework (serialization, Java RPC and persistent data storage).",
"attributes": {
"hadoop": {
"debug": true,
"cloudera_enterprise_scm": "false",
"admin_ip_eval": "Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, \"admin\").address",
"admin_interface_eval": "Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, \"admin\").interface",
"core": {
"fs_checkpoint_dir": [ "/tmp/hadoop-metadata" ],
"fs_checkpoint_edits_dir": [ "/tmp/hadoop-metadata" ],
"fs_checkpoint_period": "3600",
"fs_checkpoint_size": "67108864",
"fs_default_name": "file:///",
"fs_file_impl": "org.apache.hadoop.fs.LocalFileSystem",
"fs_ftp_impl": "org.apache.hadoop.fs.ftp.FTPFileSystem",
"fs_har_impl": "org.apache.hadoop.fs.HarFileSystem",
"fs_har_impl_disable_cache": "true",
"fs_hdfs_impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
"fs_hftp_impl": "org.apache.hadoop.hdfs.HftpFileSystem",
"fs_hsftp_impl": "org.apache.hadoop.hdfs.HsftpFileSystem",
"fs_kfs_impl": "org.apache.hadoop.fs.kfs.KosmosFileSystem",
"fs_ramfs_impl": "org.apache.hadoop.fs.InMemoryFileSystem",
"fs_s3_block_size": "67108864",
"fs_s3_buffer_dir": "/tmp/hadoop-crowbar/s3",
"fs_s3_impl": "org.apache.hadoop.fs.s3.S3FileSystem",
"fs_s3_maxRetries": "4",
"fs_s3_sleepTimeSeconds": "10",
"fs_s3n_impl": "org.apache.hadoop.fs.s3native.NativeS3FileSystem",
"fs_trash_interval": "1440",
"hadoop_http_filter_initializers": "",
"hadoop_logfile_count": "10",
"hadoop_logfile_size": "10000000",
"hadoop_native_lib": "true",
"hadoop_rpc_socket_factory_class_ClientProtocol": "",
"hadoop_rpc_socket_factory_class_default": "org.apache.hadoop.net.StandardSocketFactory",
"hadoop_security_authentication": "simple",
"hadoop_security_authorization": "false",
"hadoop_security_group_mapping": "org.apache.hadoop.security.ShellBasedUnixGroupsMapping",
"hadoop_security_uid_cache_secs": "14400",
"hadoop_socks_server": "",
"hadoop_tmp_dir": "/tmp/hadoop-crowbar",
"hadoop_util_hash_type": "murmur",
"io_bytes_per_checksum": "512",
"io_compression_codecs": "org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec",
"io_file_buffer_size": "65536",
"io_mapfile_bloom_error_rate": "0.005",
"io_mapfile_bloom_size": "1048576",
"io_seqfile_compress_blocksize": "1000000",
"io_seqfile_lazydecompress": "true",
"io_seqfile_sorter_recordlimit": "1000000",
"io_serializations": "org.apache.hadoop.io.serializer.WritableSerialization",
"io_skip_checksum_errors": "false",
"ipc_client_connect_max_retries": "10",
"ipc_client_connection_maxidletime": "10000",
"ipc_client_idlethreshold": "4000",
"ipc_client_kill_max": "10",
"ipc_client_tcpnodelay": "false",
"ipc_server_listen_queue_size": "128",
"ipc_server_tcpnodelay": "false",
"local_cache_size": "10737418240",
"topology_node_switch_mapping_impl": "org.apache.hadoop.net.ScriptBasedMapping",
"topology_script_file_name": "",
"topology_script_number_args": "100",
"webinterface_private_actions": "false"
},
"env": {
"hadoop_datanode_opts": "-Xmx2048m",
"hadoop_heapsize": "1000",
"hadoop_jobtracker_opts": "-Xmx2048m",
"hadoop_balancer_opts": "-Xmx2048m",
"hadoop_log_dir": "/var/log/hadoop",
"hadoop_namenode_opts": "-Xmx2048m",
"hadoop_secondarynamenode_opts": "-Xmx2048m",
"hadoop_tasktracker_opts": "-Xmx2048m"
},
"hdfs": {
"dfs_access_time_precision": "3600000",
"dfs_balance_bandwidthPerSec": "1048576",
"dfs_block_access_key_update_interval": "600",
"dfs_block_access_token_enable": "false",
"dfs_block_access_token_lifetime": "600",
"dfs_block_size": "134217728",
"dfs_blockreport_initialDelay": "",
"dfs_blockreport_intervalMsec": "3600000",
"dfs_client_block_write_retries": "3",
"dfs_data_dir": [ "/mnt/hdfs/hdfs01/data1" ],
"dfs_datanode_address": "0.0.0.0:50010",
"dfs_datanode_data_dir_perm": "755",
"dfs_datanode_dns_interface": "default",
"dfs_datanode_dns_nameserver": "default",
"dfs_datanode_du_reserved": "10737418240",
"dfs_datanode_failed_volumes_tolerated": "",
"dfs_datanode_handler_count": "16",
"dfs_datanode_http_address": "0.0.0.0:50075",
"dfs_datanode_https_address": "0.0.0.0:50475",
"dfs_datanode_ipc_address": "0.0.0.0:50020",
"dfs_default_chunk_view_size": "32768",
"dfs_df_interval": "60000",
"dfs_heartbeat_interval": "3",
"dfs_hosts": "",
"dfs_hosts_exclude": "",
"dfs_http_address": "0.0.0.0:50070",
"dfs_https_address": "0.0.0.0:50470",
"dfs_https_client_keystore_resource": "ssl-client.xml",
"dfs_https_enable": "false",
"dfs_https_need_client_auth": "false",
"dfs_https_server_keystore_resource": "ssl-server.xml",
"dfs_max_objects": "",
"dfs_name_dir": [ "/mnt/hdfs/hdfs01/meta1" ],
"dfs_name_edits_dir": [ "/mnt/hdfs/hdfs01/meta1" ],
"dfs_namenode_decommission_interval": "30",
"dfs_namenode_decommission_nodes_per_interval": "5",
"dfs_namenode_delegation_key_update_interval": "86400000",
"dfs_namenode_delegation_token_max_lifetime": "604800000",
"dfs_namenode_delegation_token_renew_interval": "86400000",
"dfs_namenode_handler_count": "32",
"dfs_namenode_logging_level": "info",
"dfs_permissions": "true",
"dfs_permissions_supergroup": "supergroup",
"dfs_replication": "3",
"dfs_replication_considerLoad": "true",
"dfs_replication_interval": "3",
"dfs_replication_max": "512",
"dfs_replication_min": "1",
"dfs_safemode_extension": "30000",
"dfs_safemode_threshold_pct": "0.999f",
"dfs_secondary_http_address": "0.0.0.0:50090",
"dfs_support_append": "false",
"dfs_web_ugi": "webuser,webgroup"
},
"mapred": {
"hadoop_job_history_location": "",
"hadoop_job_history_user_location": "",
"hadoop_rpc_socket_factory_class_JobSubmissionProtocol": "",
"io_map_index_skip": "",
"io_sort_factor": "80",
"io_sort_mb": "512",
"io_sort_record_percent": "0.05",
"io_sort_spill_percent": "0.80",
"job_end_retry_attempts": "",
"job_end_retry_interval": "30000",
"jobclient_output_filter": "FAILED",
"keep_failed_task_files": "false",
"map_sort_class": "org.apache.hadoop.util.QuickSort",
"mapred_acls_enabled": "false",
"mapred_child_env": "",
"mapred_child_java_opts": "-Xmx1024m",
"mapred_child_tmp": "./tmp",
"mapred_child_ulimit": "2097152",
"mapred_cluster_map_memory_mb": "-1",
"mapred_cluster_max_map_memory_mb": "-1",
"mapred_cluster_max_reduce_memory_mb": "-1",
"mapred_cluster_reduce_memory_mb": "-1",
"mapred_compress_map_output": "false",
"mapred_healthChecker_interval": "60000",
"mapred_healthChecker_script_args": "",
"mapred_healthChecker_script_path": "",
"mapred_healthChecker_script_timeout": "600000",
"mapred_heartbeats_in_second": "100",
"mapred_hosts": "",
"mapred_hosts_exclude": "",
"mapred_inmem_merge_threshold": "1000",
"mapred_job_map_memory_mb": "-1",
"mapred_job_queue_name": "default",
"mapred_job_reduce_input_buffer_percent": "0.0",
"mapred_job_reduce_memory_mb": "-1",
"mapred_job_reuse_jvm_num_tasks": "1",
"mapred_job_shuffle_input_buffer_percent": "0.70",
"mapred_job_shuffle_merge_percent": "0.66",
"mapred_job_tracker": "local",
"mapred_job_tracker_handler_count": "32",
"mapred_job_tracker_history_completed_location": "",
"mapred_job_tracker_http_address": "0.0.0.0:50030",
"mapred_job_tracker_jobhistory_lru_cache_size": "5",
"mapred_job_tracker_persist_jobstatus_active": "false",
"mapred_job_tracker_persist_jobstatus_dir": "/jobtracker/jobsInfo",
"mapred_job_tracker_persist_jobstatus_hours": "",
"mapred_job_tracker_retiredjobs_cache_size": "1000",
"mapred_jobtracker_blacklist_fault_bucket_width": "15",
"mapred_jobtracker_blacklist_fault_timeout_window": "180",
"mapred_jobtracker_completeuserjobs_maximum": "100",
"mapred_jobtracker_job_history_block_size": "3145728",
"mapred_jobtracker_maxtasks_per_job": "-1",
"mapred_jobtracker_restart_recover": "false",
"mapred_jobtracker_taskScheduler": "org.apache.hadoop.mapred.JobQueueTaskScheduler",
"mapred_jobtracker_taskScheduler_maxRunningTasksPerJob": "",
"mapred_line_input_format_linespermap": "1",
"mapred_local_dir": [ "/var/lib/hadoop-0.20/cache/mapred/mapred/local" ],
"mapred_local_dir_minspacekill": "",
"mapred_local_dir_minspacestart": "",
"mapred_map_max_attempts": "4",
"mapred_map_output_compression_codec": "org.apache.hadoop.io.compress.DefaultCodec",
"mapred_map_tasks": "2",
"mapred_map_tasks_speculative_execution": "false",
"mapred_max_tracker_blacklists": "4",
"mapred_max_tracker_failures": "4",
"mapred_merge_recordsBeforeProgress": "10000",
"mapred_min_split_size": "",
"mapred_output_compress": "false",
"mapred_output_compression_codec": "org.apache.hadoop.io.compress.DefaultCodec",
"mapred_output_compression_type": "RECORD",
"mapred_queue_default_state": "RUNNING",
"mapred_queue_names": "default",
"mapred_reduce_copy_backoff": "300",
"mapred_reduce_max_attempts": "4",
"mapred_reduce_parallel_copies": "5",
"mapred_reduce_slowstart_completed_maps": "0.05",
"mapred_reduce_tasks": "6",
"mapred_reduce_tasks_speculative_execution": "false",
"mapred_skip_attempts_to_start_skipping": "2",
"mapred_skip_map_auto_incr_proc_count": "true",
"mapred_skip_map_max_skip_records": "",
"mapred_skip_out_dir": "",
"mapred_skip_reduce_auto_incr_proc_count": "true",
"mapred_skip_reduce_max_skip_groups": "",
"mapred_submit_replication": "10",
"mapred_system_dir": "/mapred/system",
"mapred_task_cache_levels": "2",
"mapred_task_profile": "false",
"mapred_task_profile_maps": "0-2",
"mapred_task_profile_reduces": "0-2",
"mapred_task_timeout": "600000",
"mapred_task_tracker_http_address": "0.0.0.0:50060",
"mapred_task_tracker_report_address": "127.0.0.1:0",
"mapred_task_tracker_task_controller": "org.apache.hadoop.mapred.DefaultTaskController",
"mapred_tasktracker_dns_interface": "default",
"mapred_tasktracker_dns_nameserver": "default",
"mapred_tasktracker_expiry_interval": "600000",
"mapred_tasktracker_indexcache_mb": "10",
"mapred_tasktracker_map_tasks_maximum": "10",
"mapred_tasktracker_memory_calculator_plugin": "",
"mapred_tasktracker_reduce_tasks_maximum": "6",
"mapred_tasktracker_taskmemorymanager_monitoring_interval": "5000",
"mapred_tasktracker_tasks_sleeptime_before_sigkill": "5000",
"mapred_temp_dir": "/mnt/hdfs/hdfs01/data1/mapred/temp",
"mapred_user_jobconf_limit": "5242880",
"mapred_userlog_limit_kb": "",
"mapred_userlog_retain_hours": "24",
"mapreduce_job_acl_modify_job": "",
"mapreduce_job_acl_view_job": "",
"mapreduce_job_complete_cancel_delegation_tokens": "true",
"mapreduce_job_counters_limit": "120",
"mapreduce_job_split_metainfo_maxsize": "10000000",
"mapreduce_jobtracker_staging_root_dir": "/mnt/hdfs/hdfs01/data1/mapred/staging",
"mapreduce_reduce_input_limit": "-1",
"mapreduce_reduce_shuffle_connect_timeout": "180000",
"mapreduce_reduce_shuffle_read_timeout": "180000",
"mapreduce_tasktracker_group": "",
"mapreduce_tasktracker_outofband_heartbeat": "false",
"tasktracker_http_threads": "40"
},
"scheduler": {
"default_min_share_preemption_timeout": "600",
"default_pool_scheduling_mode": "fair",
"fair_share_preemption_timeout": "600",
"pool_max_jobs_default": "20",
"user_max_jobs_default": "10"
}
}
},
"deployment": {
"hadoop": {
"crowbar-revision": 0,
"element_states": {
"hadoop-masternamenode": [ "readying", "ready", "applying" ],
"hadoop-secondarynamenode": [ "readying", "ready", "applying" ],
"hadoop-slavenode": [ "readying", "ready", "applying" ],
"hadoop-edgenode": [ "readying", "ready", "applying" ]
},
"elements": {},
"element_order": [
[
"hadoop-masternamenode",
"hadoop-secondarynamenode",
"hadoop-slavenode",
"hadoop-edgenode"
]
],
"config": {
"environment": "hadoop-base-config",
"mode": "full",
"transitions": true,
"transition_list": [
"installed"
]
}
}
}
}