Hadoop3 50070端口访问不到HDFS页面
hadoop3配置注意事项：https://blog.csdn.net/u014646662/article/details/82890443
hadoop3 hdfs web端口改为9870
<property>
<name>dfs.namenode.http-address</name>
<value>0.0.0.0:9870</value>
<description>
The address and the base port where the dfs namenode web ui will listen on.
</description>
</property>
下面是hadoop的默认配置
1.core-default.xml
<xml version="1.0">
<xml-stylesheet type="text/xsl" href="configuration.xsl">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Do not modify this file directly. Instead, copy entries that you -->
<!-- wish to modify from this file into core-site.xml and change them -->
<!-- there. If core-site.xml does not already exist, create it. -->
<configuration>
<!--- global properties -->
<property>
<name>hadoop.common.configuration.version</name>
<value>3.0.0</value>
<description>version of this configuration file</description>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/tmp/hadoop-${user.name}</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>hadoop.http.filter.initializers</name>
<value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
<description>A comma separated list of class names. Each class in the list
must extend org.apache.hadoop.http.FilterInitializer. The corresponding
Filter will be initialized. Then, the Filter will be applied to all user
facing jsp and servlet web pages. The ordering of the list defines the
ordering of the filters.</description>
</property>
<!--- security properties -->
<property>
<name>hadoop.security.authorization</name>
<value>false</value>
<description>Is service-level authorization enabled</description>
</property>
<property>
<name>hadoop.security.instrumentation.requires.admin</name>
<value>false</value>
<description>
Indicates if administrator ACLs are required to access
instrumentation servlets (JMX, METRICS, CONF, STACKS).
</description>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
<description>Possible values are simple (no authentication), and kerberos
</description>
</property>
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback</value>
<description>
Class for user to group mapping (get groups for a given user) for ACL.
The default implementation,
org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
will determine if the Java Native Interface (JNI) is available. If JNI is
available the implementation will use the API within hadoop to resolve a
list of groups for a user. If JNI is not available then the shell
implementation, ShellBasedUnixGroupsMapping, is used. This implementation
shells out to the Linux/Unix environment with the
<code>bash -c groups</code> command to resolve a list of groups for a user.
</description>
</property>
<property>
<name>hadoop.security.dns.interface</name>
<description>
The name of the Network Interface from which the service should determine
its host name for Kerberos login. e.g. eth2. In a multi-homed environment,
the setting can be used to affect the _HOST substitution in the service
Kerberos principal. If this configuration value is not set, the service
will use its default hostname as returned by
InetAddress.getLocalHost().getCanonicalHostName().
Most clusters will not require this setting.
</description>
</property>
<property>
<name>hadoop.security.dns.nameserver</name>
<description>
The host name or IP address of the name server (DNS) which a service Node
should use to determine its own host name for Kerberos Login. Requires
hadoop.security.dns.interface.
Most clusters will not require this setting.
</description>
</property>
<property>
<name>hadoop.security.dns.log-slow-lookups.enabled</name>
<value>false</value>
<description>
Time name lookups (via SecurityUtil) and log them if they exceed the
configured threshold.
</description>
</property>
<property>
<name>hadoop.security.dns.log-slow-lookups.threshold.ms</name>
<value>1000</value>
<description>
If slow lookup logging is enabled, this threshold is used to decide if a
lookup is considered slow enough to be logged.
</description>
</property>
<property>
<name>hadoop.security.groups.cache.secs</name>
<value>300</value>
<description>
This is the config controlling the validity of the entries in the cache
containing the user->group mapping. When this duration has expired,
then the implementation of the group mapping provider is invoked to get
the groups of the user and then cached back.
</description>
</property>
<property>
<name>hadoop.security.groups.negative-cache.secs</name>
<value>30</value>
<description>
Expiration time for entries in the the negative user-to-group mapping
caching, in seconds. This is useful when invalid users are retrying
frequently. It is suggested to set a small value for this expiration, since
a transient error in group lookup could temporarily lock out a legitimate
user.
Set this to zero or negative value to disable negative user-to-group caching.
</description>
</property>
<property>
<name>hadoop.security.groups.cache.warn.after.ms</name>
<value>5000</value>
<description>
If looking up a single user to group takes longer than this amount of
milliseconds, we will log a warning message.
</description>
</property>
<property>
<name>hadoop.security.groups.cache.background.reload</name>
<value>false</value>
<description>
Whether to reload expired user->group mappings using a background thread
pool. If set to true, a pool of
hadoop.security.groups.cache.background.reload.threads is created to
update the cache in the background.
</description>
</property>
<property>
<name>hadoop.security.groups.cache.background.reload.threads</name>
<value>3</value>
<description>
Only relevant if hadoop.security.groups.cache.background.reload is true.
Controls the number of concurrent background user->group cache entry
refreshes. Pending refresh requests beyond this value are queued and
processed when a thread is free.
</description>
</property>
<property>
<name>hadoop.security.groups.shell.command.timeout</name>
<value>0s</value>
<description>
Used by the ShellBasedUnixGroupsMapping class, this property controls how
long to wait for the underlying shell command that is run to fetch groups.
Expressed in seconds (e.g. 10s, 1m, etc.), if the running command takes
longer than the value configured, the command is aborted and the groups
resolver would return a result of no groups found. A value of 0s (default)
would mean an infinite wait (i.e. wait until the command exits on its own).
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.connection.timeout.ms</name>
<value>60000</value>
<description>
This property is the connection timeout (in milliseconds) for LDAP
operations. If the LDAP provider doesn't establish a connection within the
specified period, it will abort the connect attempt. Non-positive value
means no LDAP connection timeout is specified in which case it waits for the
connection to establish until the underlying network times out.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.read.timeout.ms</name>
<value>60000</value>
<description>
This property is the read timeout (in milliseconds) for LDAP
operations. If the LDAP provider doesn't get a LDAP response within the
specified period, it will abort the read attempt. Non-positive value
means no read timeout is specified in which case it waits for the response
infinitely.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.url</name>
<value></value>
<description>
The URL of the LDAP server to use for resolving user groups when using
the LdapGroupsMapping user to group mapping.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl</name>
<value>false</value>
<description>
Whether or not to use SSL when connecting to the LDAP server.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl.keystore</name>
<value></value>
<description>
File path to the SSL keystore that contains the SSL certificate required
by the LDAP server.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl.keystore.password.file</name>
<value></value>
<description>
The path to a file containing the password of the LDAP SSL keystore. If
the password is not configured in credential providers and the property
hadoop.security.group.mapping.ldap.ssl.keystore.password is not set,
LDAPGroupsMapping reads password from the file.
IMPORTANT: This file should be readable only by the Unix user running
the daemons and should be a local file.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl.keystore.password</name>
<value></value>
<description>
The password of the LDAP SSL keystore. this property name is used as an
alias to get the password from credential providers. If the password can
not be found and hadoop.security.credential.clear-text-fallback is true
LDAPGroupsMapping uses the value of this property for password.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.conversion.rule</name>
<value>none</value>
<description>
The rule is applied on the group names received from LDAP when
RuleBasedLdapGroupsMapping is configured.
Supported rules are "to_upper", "to_lower" and "none".
to_upper: This will convert all the group names to uppercase.
to_lower: This will convert all the group names to lowercase.
none: This will retain the source formatting, this is default value.
</description>
</property>
<property>
<name>hadoop.security.credential.clear-text-fallback</name>
<value>true</value>
<description>
true or false to indicate whether or not to fall back to storing credential
password as clear text. The default value is true. This property only works
when the password can't not be found from credential providers.
</description>
</property>
<property>
<name>hadoop.security.credential.provider.path</name>
<value></value>
<description>
A comma-separated list of URLs that indicates the type and
location of a list of providers that should be consulted.
</description>
</property>
<property>
<name>hadoop.security.credstore.java-keystore-provider.password-file</name>
<value></value>
<description>
The path to a file containing the custom password for all keystores
that may be configured in the provider path.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl.truststore</name>
<value></value>
<description>
File path to the SSL truststore that contains the root certificate used to
sign the LDAP server's certificate. Specify this if the LDAP server's
certificate is not signed by a well known certificate authority.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.ssl.truststore.password.file</name>
<value></value>
<description>
The path to a file containing the password of the LDAP SSL truststore.
IMPORTANT: This file should be readable only by the Unix user running
the daemons.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.bind.user</name>
<value></value>
<description>
The distinguished name of the user to bind as when connecting to the LDAP
server. This may be left blank if the LDAP server supports anonymous binds.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.bind.password.file</name>
<value></value>
<description>
The path to a file containing the password of the bind user. If
the password is not configured in credential providers and the property
hadoop.security.group.mapping.ldap.bind.password is not set,
LDAPGroupsMapping reads password from the file.
IMPORTANT: This file should be readable only by the Unix user running
the daemons and should be a local file.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.bind.password</name>
<value></value>
<description>
The password of the bind user. this property name is used as an
alias to get the password from credential providers. If the password can
not be found and hadoop.security.credential.clear-text-fallback is true
LDAPGroupsMapping uses the value of this property for password.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.base</name>
<value></value>
<description>
The search base for the LDAP connection. This is a distinguished name,
and will typically be the root of the LDAP directory.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.userbase</name>
<value></value>
<description>
The search base for the LDAP connection for user search query. This is a
distinguished name, and its the root of the LDAP directory for users.
If not set, hadoop.security.group.mapping.ldap.base is used.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.groupbase</name>
<value></value>
<description>
The search base for the LDAP connection for group search . This is a
distinguished name, and its the root of the LDAP directory for groups.
If not set, hadoop.security.group.mapping.ldap.base is used.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.filter.user</name>
<value>(&(objectClass=user)(sAMAccountName={0}))</value>
<description>
An additional filter to use when searching for LDAP users. The default will
usually be appropriate for Active Directory installations. If connecting to
an LDAP server with a non-AD schema, this should be replaced with
(&(objectClass=inetOrgPerson)(uid={0}). {0} is a special string used to
denote where the username fits into the filter.
If the LDAP server supports posixGroups, Hadoop can enable the feature by
setting the value of this property to "posixAccount" and the value of
the hadoop.security.group.mapping.ldap.search.filter.group property to
"posixGroup".
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.filter.group</name>
<value>(objectClass=group)</value>
<description>
An additional filter to use when searching for LDAP groups. This should be
changed when resolving groups against a non-Active Directory installation.
See the description of hadoop.security.group.mapping.ldap.search.filter.user
to enable posixGroups support.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.attr.memberof</name>
<value></value>
<description>
The attribute of the user object that identifies its group objects. By
default, Hadoop makes two LDAP queries per user if this value is empty. If
set, Hadoop will attempt to resolve group names from this attribute,
instead of making the second LDAP query to get group objects. The value
should be 'memberOf' for an MS AD installation.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.attr.member</name>
<value>member</value>
<description>
The attribute of the group object that identifies the users that are
members of the group. The default will usually be appropriate for
any LDAP installation.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.attr.group.name</name>
<value>cn</value>
<description>
The attribute of the group object that identifies the group name. The
default will usually be appropriate for all LDAP systems.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.search.group.hierarchy.levels</name>
<value>0</value>
<description>
The number of levels to go up the group hierarchy when determining
which groups a user is part of. 0 Will represent checking just the
group that the user belongs to. Each additional level will raise the
time it takes to execute a query by at most
hadoop.security.group.mapping.ldap.directory.search.timeout.
The default will usually be appropriate for all LDAP systems.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.posix.attr.uid.name</name>
<value>uidNumber</value>
<description>
The attribute of posixAccount to use when groups for membership.
Mostly useful for schemas wherein groups have memberUids that use an
attribute other than uidNumber.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.posix.attr.gid.name</name>
<value>gidNumber</value>
<description>
The attribute of posixAccount indicating the group id.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.ldap.directory.search.timeout</name>
<value>10000</value>
<description>
The attribute applied to the LDAP SearchControl properties to set a
maximum time limit when searching and awaiting a result.
Set to 0 if infinite wait period is desired.
Default is 10 seconds. Units in milliseconds.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.providers</name>
<value></value>
<description>
Comma separated of names of other providers to provide user to group
mapping. Used by CompositeGroupsMapping.
</description>
</property>
<property>
<name>hadoop.security.group.mapping.providers.combined</name>
<value>true</value>
<description>
true or false to indicate whether groups from the providers are combined or
not. The default value is true. If true, then all the providers will be
tried to get groups and all the groups are combined to return as the final
results. Otherwise, providers are tried one by one in the configured list
order, and if any groups are retrieved from any provider, then the groups
will be returned without trying the left ones.
</description>
</property>
<property>
<name>hadoop.security.service.user.name.key</name>
<value></value>
<description>
For those cases where the same RPC protocol is implemented by multiple
servers, this configuration is required for specifying the principal
name to use for the service when the client wishes to make an RPC call.
</description>
</property>
<property>
<name>fs.azure.user.agent.prefix</name>
<value>unknown</value>
<description>
WASB passes User-Agent header to the Azure back-end. The default value
contains WASB version, Java Runtime version, Azure Client library version,
and the value of the configuration option fs.azure.user.agent.prefix.
</description>
</property>
<property>
<name>hadoop.security.uid.cache.secs</name>
<value>14400</value>
<description>
This is the config controlling the validity of the entries in the cache
containing the userId to userName and groupId to groupName used by
NativeIO getFstat().
</description>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication</value>
<description>A comma-separated list of protection values for secured sasl
connections. Possible values are authentication, integrity and privacy.
authentication means authentication only and no integrity or privacy;
integrity implies authentication and integrity are enabled; and privacy
implies all of authentication, integrity and privacy are enabled.
hadoop.security.saslproperties.resolver.class can be used to override
the hadoop.rpc.protection for a connection at the server side.
</description>
</property>
<property>
<name>hadoop.security.saslproperties.resolver.class</name>
<value></value>
<description>SaslPropertiesResolver used to resolve the QOP used for a
connection. If not specified, the full set of values specified in
hadoop.rpc.protection is used while determining the QOP used for the
connection. If a class is specified, then the QOP values returned by
the class will be used while determining the QOP used for the connection.
</description>
</property>
<property>
<name>hadoop.security.sensitive-config-keys</name>
<value>
secret$
password$
ssl.keystore.pass$
fs.s3.*[Ss]ecret.[Kk]ey
fs.s3a.*.server-side-encryption.key
fs.azure.account.key.*
credential$
oauth.*token$
hadoop.security.sensitive-config-keys
</value>
<description>A comma-separated or multi-line list of regular expressions to
match configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration,
private credentials should not be logged.
</description>
</property>
<property>
<name>hadoop.workaround.non.threadsafe.getpwuid</name>
<value>true</value>
<description>Some operating systems or authentication modules are known to
have broken implementations of getpwuid_r and getpwgid_r, such that these
calls are not thread-safe. Symptoms of this problem include JVM crashes
with a stack trace inside these functions. If your system exhibits this
issue, enable this configuration parameter to include a lock around the
calls as a workaround.
An incomplete list of some systems known to have this issue is available
at http://wiki.apache.org/hadoop/KnownBrokenPwuidImplementations
</description>
</property>
<property>
<name>hadoop.kerberos.kinit.command</name>
<value>kinit</value>
<description>Used to periodically renew Kerberos credentials when provided
to Hadoop. The default setting assumes that kinit is in the PATH of users
running the Hadoop client. Change this to the absolute path to kinit if this
is not the case.
</description>
</property>
<property>
<name>hadoop.kerberos.min.seconds.before.relogin</name>
<value>60</value>
<description>The minimum time between relogin attempts for Kerberos, in
seconds.
</description>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value></value>
<description>Maps kerberos principals to local user names</description>
</property>
<property>
<name>hadoop.token.files</name>
<value></value>
<description>List of token cache files that have delegation tokens for hadoop service</description>
</property>
<!-- i/o properties -->
<property>
<name>io.file.buffer.size</name>
<value>4096</value>
<description>The size of buffer for use in sequence files.
The size of this buffer should probably be a multiple of hardware
page size (4096 on Intel x86), and it determines how much data is
buffered during read and write operations.</description>
</property>
<property>
<name>io.bytes.per.checksum</name>
<value>512</value>
<description>The number of bytes per checksum. Must not be larger than
io.file.buffer.size.</description>
</property>
<property>
<name>io.skip.checksum.errors</name>
<value>false</value>
<description>If true, when a checksum error is encountered while
reading a sequence file, entries are skipped, instead of throwing an
exception.</description>
</property>
<property>
<name>io.compression.codecs</name>
<value></value>
<description>A comma-separated list of the compression codec classes that can
be used for compression/decompression. In addition to any classes specified
with this property (which take precedence), codec classes on the classpath
are discovered using a Java ServiceLoader.</description>
</property>
<property>
<name>io.compression.codec.bzip2.library</name>
<value>system-native</value>
<description>The native-code library to be used for compression and
decompression by the bzip2 codec. This library could be specified
either by by name or the full pathname. In the former case, the
library is located by the dynamic linker, usually searching the
directories specified in the environment variable LD_LIBRARY_PATH.
The value of "system-native" indicates that the default system
library should be used. To indicate that the algorithm should
operate entirely in Java, specify "java-builtin".</description>
</property>
<property>
<name>io.serializations</name>
<value>org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>
<description>A list of serialization classes that can be used for
obtaining serializers and deserializers.</description>
</property>
<property>
<name>io.seqfile.local.dir</name>
<value>${hadoop.tmp.dir}/io/local</value>
<description>The local directory where sequence file stores intermediate
data files during merge. May be a comma-separated list of
directories on different devices in order to spread disk i/o.
Directories that do not exist are ignored.
</description>
</property>
<property>
<name>io.map.index.skip</name>
<value>0</value>
<description>Number of index entries to skip between each entry.
Zero by default. Setting this to values larger than zero can
facilitate opening large MapFiles using less memory.</description>
</property>
<property>
<name>io.map.index.interval</name>
<value>128</value>
<description>
MapFile consist of two files - data file (tuples) and index file
(keys). For every io.map.index.interval records written in the
data file, an entry (record-key, data-file-position) is written
in the index file. This is to allow for doing binary search later
within the index file to look up records by their keys and get their
closest positions in the data file.
</description>
</property>
<property>
<name>io.erasurecode.codec.rs.rawcoders</name>
<value>rs_native,rs_java</value>
<description>
Comma separated raw coder implementations for the rs codec. The earlier
factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
<property>
<name>io.erasurecode.codec.rs-legacy.rawcoders</name>
<value>rs-legacy_java</value>
<description>
Comma separated raw coder implementations for the rs-legacy codec. The earlier
factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
<property>
<name>io.erasurecode.codec.xor.rawcoders</name>
<value>xor_native,xor_java</value>
<description>
Comma separated raw coder implementations for the xor codec. The earlier
factory is prior to followings in case of failure of creating raw coders.
</description>
</property>
<!-- file system properties -->
<property>
<name>fs.defaultFS</name>
<value>file:///</value>
<description>The name of the default file system. A URI whose
scheme and authority determine the FileSystem implementation. The
uri's scheme determines the config property (fs.SCHEME.impl) naming
the FileSystem implementation class. The uri's authority is used to
determine the host, port, etc. for a filesystem.</description>
</property>
<property>
<name>fs.default.name</name>
<value>file:///</value>
<description>Deprecated. Use (fs.defaultFS) property
instead</description>
</property>
<property>
<name>fs.trash.interval</name>
<value>0</value>
<description>Number of minutes after which the checkpoint
gets deleted. If zero, the trash feature is disabled.
This option may be configured both on the server and the
client. If trash is disabled server side then the client
side configuration is checked. If trash is enabled on the
server side then the value configured on the server is
used and the client configuration value is ignored.
</description>
</property>
<property>
<name>fs.trash.checkpoint.interval</name>
<value>0</value>
<description>Number of minutes between trash checkpoints.
Should be smaller or equal to fs.trash.interval. If zero,
the value is set to the value of fs.trash.interval.
Every time the checkpointer runs it creates a new checkpoint
out of current and removes checkpoints created more than
fs.trash.interval minutes ago.
</description>
</property>
<property>
<name>fs.protected.directories</name>
<value></value>
<description>A comma-separated list of directories which cannot
be deleted even by the superuser unless they are empty. This
setting can be used to guard important system directories
against accidental deletion due to administrator error.
</description>
</property>
<property>
<name>fs.AbstractFileSystem.file.impl</name>
<value>org.apache.hadoop.fs.local.LocalFs</value>
<description>The AbstractFileSystem for file: uris.</description>
</property>
<property>
<name>fs.AbstractFileSystem.har.impl</name>
<value>org.apache.hadoop.fs.HarFs</value>
<description>The AbstractFileSystem for har: uris.</description>
</property>
<property>
<name>fs.AbstractFileSystem.hdfs.impl</name>
<value>org.apache.hadoop.fs.Hdfs</value>
<description>The FileSystem for hdfs: uris.</description>
</property>
<property>
<name>fs.AbstractFileSystem.viewfs.impl</name>
<value>org.apache.hadoop.fs.viewfs.ViewFs</value>
<description>The AbstractFileSystem for view file system for viewfs: uris
(ie client side mount table:).</description>
</property>
<property>
<name>fs.viewfs.rename.strategy</name>
<value>SAME_MOUNTPOINT</value>
<description>Allowed rename strategy to rename between multiple mountpoints.
Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
</description>
</property>
<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
<description>The FileSystem for Ftp: uris.</description>
</property>
<property>
<name>fs.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
<description>The implementation class of the FTP FileSystem</description>
</property>
<property>
<name>fs.AbstractFileSystem.webhdfs.impl</name>
<value>org.apache.hadoop.fs.WebHdfs</value>
<description>The FileSystem for webhdfs: uris.</description>
</property>
<property>
<name>fs.AbstractFileSystem.swebhdfs.impl</name>
<value>org.apache.hadoop.fs.SWebHdfs</value>
<description>The FileSystem for swebhdfs: uris.</description>
</property>
<property>
<name>fs.ftp.host</name>
<value>0.0.0.0</value>
<description>FTP filesystem connects to this server</description>
</property>
<property>
<name>fs.ftp.host.port</name>
<value>21</value>
<description>
FTP filesystem connects to fs.ftp.host on this port
</description>
</property>
<property>
<name>fs.ftp.data.connection.mode</name>
<value>ACTIVE_LOCAL_DATA_CONNECTION_MODE</value>
<description>Set the FTPClient's data connection mode based on configuration.
Valid values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
</description>
</property>
<property>
<name>fs.ftp.transfer.mode</name>
<value>BLOCK_TRANSFER_MODE</value>
<description>
Set FTP's transfer mode based on configuration. Valid values are
STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
</description>
</property>
<property>
<name>fs.df.interval</name>
<value>60000</value>
<description>Disk usage statistics refresh interval in msec.</description>
</property>
<property>
<name>fs.du.interval</name>
<value>600000</value>
<description>File space usage statistics refresh interval in msec.</description>
</property>
<property>
<name>fs.swift.impl</name>
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
<description>The implementation class of the OpenStack Swift Filesystem</description>
</property>
<property>
<name>fs.automatic.close</name>
<value>true</value>
<description>By default, FileSystem instances are automatically closed at program
exit using a JVM shutdown hook. Setting this property to false disables this
behavior. This is an advanced option that should only be used by server applications
requiring a more carefully orchestrated shutdown sequence.
</description>
</property>
<property>
<name>fs.s3a.access.key</name>
<description>AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
</property>
<property>
<name>fs.s3a.secret.key</name>
<description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
</property>
<property>
<name>fs.s3a.aws.credentials.provider</name>
<description>
Comma-separated class names of credential provider classes which implement
com.amazonaws.auth.AWSCredentialsProvider.
These are loaded and queried in sequence for a valid set of credentials.
Each listed class must implement one of the following means of
construction, which are attempted in order:
1. a public constructor accepting java.net.URI and
org.apache.hadoop.conf.Configuration,
2. a public static method named getInstance that accepts no
arguments and returns an instance of
com.amazonaws.auth.AWSCredentialsProvider, or
3. a public default constructor.
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
anonymous access to a publicly accessible S3 bucket without any credentials.
Please note that allowing anonymous access to an S3 bucket compromises
security and therefore is unsuitable for most use cases. It can be useful
for accessing public data sets without requiring AWS credentials.
If unspecified, then the default list of credential provider classes,
queried in sequence, is:
1. org.apache.hadoop.fs.s3a.BasicAWSCredentialsProvider: supports static
configuration of AWS access key ID and secret access key. See also
fs.s3a.access.key and fs.s3a.secret.key.
2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
of instance profile credentials if running in an EC2 VM.
</description>
</property>
<property>
<name>fs.s3a.session.token</name>
<description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
as one of the providers.
</description>
</property>
<property>
<name>fs.s3a.security.credential.provider.path</name>
<value />
<description>
Optional comma separated list of credential providers, a list
which is prepended to that set in hadoop.security.credential.provider.path
</description>
</property>
<property>
<name>fs.s3a.assumed.role.arn</name>
<value />
<description>
AWS ARN for the role to be assumed.
Required if the fs.s3a.aws.credentials.provider contains
org.apache.hadoop.fs.s3a.AssumedRoleCredentialProvider
</description>
</property>
<property>
<name>fs.s3a.assumed.role.session.name</name>
<value />
<description>
Session name for the assumed role, must be valid characters according to
the AWS APIs.
Only used if AssumedRoleCredentialProvider is the AWS credential provider.
If not set, one is generated from the current Hadoop/Kerberos username.
</description>
</property>
<property>
<name>fs.s3a.assumed.role.policy</name>
<value/>
<description>
JSON policy to apply to the role.
Only used if AssumedRoleCredentialProvider is the AWS credential provider.
</description>
</property>
<property>
<name>fs.s3a.assumed.role.session.duration</name>
<value>30m</value>
<description>
Duration of assumed roles before a refresh is attempted.
Only used if AssumedRoleCredentialProvider is the AWS credential provider.
Range: 15m to 1h
</description>
</property>
<property>
<name>fs.s3a.assumed.role.sts.endpoint</name>
<value/>
<description>
AWS Simple Token Service Endpoint. If unset, uses the default endpoint.
Only used if AssumedRoleCredentialProvider is the AWS credential provider.
</description>
</property>
<property>
<name>fs.s3a.assumed.role.credentials.provider</name>
<value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
<description>
List of credential providers to authenticate with the STS endpoint and
retrieve short-lived role credentials.
Only used if AssumedRoleCredentialProvider is the AWS credential provider.
If unset, uses "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider".
</description>
</property>
<property>
<name>fs.s3a.connection.maximum</name>
<value>15</value>
<description>Controls the maximum number of simultaneous connections to S3.</description>
</property>
<property>
<name>fs.s3a.connection.ssl.enabled</name>
<value>true</value>
<description>Enables or disables SSL connections to S3.</description>
</property>
<property>
<name>fs.s3a.endpoint</name>
<description>AWS S3 endpoint to connect to. An up-to-date list is
provided in the AWS Documentation: regions and endpoints. Without this
property, the standard region (s3.amazonaws.com) is assumed.
</description>
</property>
<property>
<name>fs.s3a.path.style.access</name>
<value>false</value>
<description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
</description>
</property>
<property>
<name>fs.s3a.proxy.host</name>
<description>Hostname of the (optional) proxy server for S3 connections.</description>
</property>
<property>
<name>fs.s3a.proxy.port</name>
<description>Proxy server port. If this property is not set
but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
the value of fs.s3a.connection.ssl.enabled).</description>
</property>
<property>
<name>fs.s3a.proxy.username</name>
<description>Username for authenticating with proxy server.</description>
</property>
<property>
<name>fs.s3a.proxy.password</name>
<description>Password for authenticating with proxy server.</description>
</property>
<property>
<name>fs.s3a.proxy.domain</name>
<description>Domain for authenticating with proxy server.</description>
</property>
<property>
<name>fs.s3a.proxy.workstation</name>
<description>Workstation for authenticating with proxy server.</description>
</property>
<property>
<name>fs.s3a.attempts.maximum</name>
<value>20</value>
<description>How many times we should retry commands on transient errors.</description>
</property>
<property>
<name>fs.s3a.connection.establish.timeout</name>
<value>5000</value>
<description>Socket connection setup timeout in milliseconds.</description>
</property>
<property>
<name>fs.s3a.connection.timeout</name>
<value>200000</value>
<description>Socket connection timeout in milliseconds.</description>
</property>
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
<description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.socket.recv.buffer</name>
<value>8192</value>
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.paging.maximum</name>
<value>5000</value>
<description>How many keys to request from S3 when doing
directory listings at a time.</description>
</property>
<property>
<name>fs.s3a.threads.max</name>
<value>10</value>
<description>The total number of threads available in the filesystem for data
uploads *or any other queued filesystem operation*.</description>
</property>
<property>
<name>fs.s3a.threads.keepalivetime</name>
<value>60</value>
<description>Number of seconds a thread can be idle before being
terminated.</description>
</property>
<property>
<name>fs.s3a.max.total.tasks</name>
<value>5</value>
<description>The number of operations which can be queued for execution</description>
</property>
<property>
<name>fs.s3a.multipart.size</name>
<value>100M</value>
<description>How big (in bytes) to split upload or copy operations up into.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
<property>
<name>fs.s3a.multipart.threshold</name>
<value>2147483647</value>
<description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves
copying the source file(s).
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
<property>
<name>fs.s3a.multiobjectdelete.enable</name>
<value>true</value>
<description>When enabled, multiple single-object delete requests are replaced by
a single 'delete multiple objects'-request, reducing the number of requests.
Beware: legacy S3-compatible object stores might not support this request.
</description>
</property>
<property>
<name>fs.s3a.acl.default</name>
<description>Set a canned ACL for newly created and copied objects. Value may be Private,
PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
or BucketOwnerFullControl.</description>
</property>
<property>
<name>fs.s3a.multipart.purge</name>
<value>false</value>
<description>True if you want to purge existing multipart uploads that may not have been
completed/aborted correctly. The corresponding purge age is defined in
fs.s3a.multipart.purge.age.
If set, when the filesystem is instantiated then all outstanding uploads
older than the purge age will be terminated -across the entire bucket.
This will impact multipart uploads by other applications and users. so should
be used sparingly, with an age value chosen to stop failed uploads, without
breaking ongoing operations.
</description>
</property>
<property>
<name>fs.s3a.multipart.purge.age</name>
<value>86400</value>
<description>Minimum age in seconds of multipart uploads to purge
on startup if "fs.s3a.multipart.purge" is true
</description>
</property>
<property>
<name>fs.s3a.server-side-encryption-algorithm</name>
<description>Specify a server-side encryption algorithm for s3a: file system.
Unset by default. It supports the following values: 'AES256' (for SSE-S3),
'SSE-KMS' and 'SSE-C'.
</description>
</property>
<property>
<name>fs.s3a.server-side-encryption.key</name>
<description>Specific encryption key to use if fs.s3a.server-side-encryption-algorithm
has been set to 'SSE-KMS' or 'SSE-C'. In the case of SSE-C, the value of this property
should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty,
you'll be using your default's S3 KMS key, otherwise you should set this property to
the specific KMS key id.
</description>
</property>
<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
implementations can still be used</description>
</property>
<property>
<name>fs.s3a.block.size</name>
<value>32M</value>
<description>Block size to use when reading files using s3a: file system.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
<property>
<name>fs.s3a.buffer.dir</name>
<value>${hadoop.tmp.dir}/s3a</value>
<description>Comma separated list of directories that will be used to buffer file
uploads to.</description>
</property>
<property>
<name>fs.s3a.fast.upload.buffer</name>
<value>disk</value>
<description>
The buffering mechanism to for data being written.
Values: disk, array, bytebuffer.
"disk" will use the directories listed in fs.s3a.buffer.dir as
the location(s) to save data prior to being uploaded.
"array" uses arrays in the JVM heap
"bytebuffer" uses off-heap memory within the JVM.
Both "array" and "bytebuffer" will consume memory in a single stream up to the number
of blocks set by:
fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks.
If using either of these mechanisms, keep this value low
The total number of threads performing work across all threads is set by
fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued
work items.
</description>
</property>
<property>
<name>fs.s3a.fast.upload.active.blocks</name>
<value>4</value>
<description>
Maximum Number of blocks a single output stream can have
active (uploading, or queued to the central FileSystem
instance's pool of queued operations.
This stops a single stream overloading the shared thread pool.
</description>
</property>
<property>
<name>fs.s3a.readahead.range</name>
<value>64K</value>
<description>Bytes to read ahead during a seek() before closing and
re-opening the S3 HTTP connection. This option will be overridden if
any call to setReadahead() is made to an open stream.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
<property>
<name>fs.s3a.user.agent.prefix</name>
<value></value>
<description>
Sets a custom value that will be prepended to the User-Agent header sent in
HTTP requests to the S3 back-end by S3AFileSystem. The User-Agent header
always includes the Hadoop version number followed by a string generated by
the AWS SDK. An example is "User-Agent: Hadoop 2.8.0, aws-sdk-java/1.10.6".
If this optional property is set, then its value is prepended to create a
customized User-Agent. For example, if this configuration property was set
to "MyApp", then an example of the resulting User-Agent would be
"User-Agent: MyApp, Hadoop 2.8.0, aws-sdk-java/1.10.6".
</description>
</property>
<property>
<name>fs.s3a.metadatastore.authoritative</name>
<value>false</value>
<description>
When true, allow MetadataStore implementations to act as source of
truth for getting file status and directory listings. Even if this
is set to true, MetadataStore implementations may choose not to
return authoritative results. If the configured MetadataStore does
not support being authoritative, this setting will have no effect.
</description>
</property>
<property>
<name>fs.s3a.metadatastore.impl</name>
<value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
<description>
Fully-qualified name of the class that implements the MetadataStore
to be used by s3a. The default class, NullMetadataStore, has no
effect: s3a will continue to treat the backing S3 service as the one
and only source of truth for file and directory metadata.
</description>
</property>
<property>
<name>fs.s3a.s3guard.cli.prune.age</name>
<value>86400000</value>
<description>
Default age (in milliseconds) after which to prune metadata from the
metadatastore when the prune command is run. Can be overridden on the
command-line.
</description>
</property>
<property>
<name>fs.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
<description>The implementation class of the S3A Filesystem</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.region</name>
<value></value>
<description>
AWS DynamoDB region to connect to. An up-to-date list is
provided in the AWS Documentation: regions and endpoints. Without this
property, the S3Guard will operate table in the associated S3 bucket region.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.table</name>
<value></value>
<description>
The DynamoDB table name to operate. Without this property, the respective
S3 bucket name will be used.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.table.create</name>
<value>false</value>
<description>
If true, the S3A client will create the table if it does not already exist.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.table.capacity.read</name>
<value>500</value>
<description>
Provisioned throughput requirements for read operations in terms of capacity
units for the DynamoDB table. This config value will only be used when
creating a new DynamoDB table, though later you can manually provision by
increasing or decreasing read capacity as needed for existing tables.
See DynamoDB documents for more information.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.table.capacity.write</name>
<value>100</value>
<description>
Provisioned throughput requirements for write operations in terms of
capacity units for the DynamoDB table. Refer to related config
fs.s3a.s3guard.ddb.table.capacity.read before usage.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.max.retries</name>
<value>9</value>
<description>
Max retries on batched DynamoDB operations before giving up and
throwing an IOException. Each retry is delayed with an exponential
backoff timer which starts at 100 milliseconds and approximately
doubles each time. The minimum wait before throwing an exception is
sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
So N = 9 yields at least 51.1 seconds (51,100) milliseconds of blocking
before throwing an IOException.
</description>
</property>
<property>
<name>fs.s3a.s3guard.ddb.background.sleep</name>
<value>25</value>
<description>
Length (in milliseconds) of pause between each batch of deletes when
pruning metadata. Prevents prune operations (which can typically be low
priority background operations) from overly interfering with other I/O
operations.
</description>
</property>
<property>
<name>fs.s3a.retry.limit</name>
<value>${fs.s3a.attempts.maximum}</value>
<description>
Number of times to retry any repeatable S3 client request on failure,
excluding throttling requests.
</description>
</property>
<property>
<name>fs.s3a.retry.interval</name>
<value>500ms</value>
<description>
Interval between attempts to retry operations for any reason other
than S3 throttle errors.
</description>
</property>
<property>
<name>fs.s3a.retry.throttle.limit</name>
<value>${fs.s3a.attempts.maximum}</value>
<description>
Number of times to retry any throttled request.
</description>
</property>
<property>
<name>fs.s3a.retry.throttle.interval</name>
<value>1000ms</value>
<description>
Interval between retry attempts on throttled requests.
</description>
</property>
<property>
<name>fs.s3a.committer.name</name>
<value>file</value>
<description>
Committer to create for output to S3A, one of:
"file", "directory", "partitioned", "magic".
</description>
</property>
<property>
<name>fs.s3a.committer.magic.enabled</name>
<value>false</value>
<description>
Enable support in the filesystem for the S3 "Magic" committer.
When working with AWS S3, S3Guard must be enabled for the destination
bucket, as consistent metadata listings are required.
</description>
</property>
<property>
<name>fs.s3a.committer.threads</name>
<value>8</value>
<description>
Number of threads in committers for parallel operations on files
(upload, commit, abort, delete...)
</description>
</property>
<property>
<name>fs.s3a.committer.staging.tmp.path</name>
<value>tmp/staging</value>
<description>
Path in the cluster filesystem for temporary data.
This is for HDFS, not the local filesystem.
It is only for the summary data of each file, not the actual
data being committed.
Using an unqualified path guarantees that the full path will be
generated relative to the home directory of the user creating the job,
hence private (assuming home directory permissions are secure).
</description>
</property>
<property>
<name>fs.s3a.committer.staging.unique-filenames</name>
<value>true</value>
<description>
Option for final files to have a unique name through job attempt info,
or the value of fs.s3a.committer.staging.uuid
When writing data with the "append" conflict option, this guarantees
that new data will not overwrite any existing data.
</description>
</property>
<property>
<name>fs.s3a.committer.staging.conflict-mode</name>
<value>fail</value>
<description>
Staging committer conflict resolution policy.
Supported: "fail", "append", "replace".
</description>
</property>
<property>
<name>fs.s3a.committer.staging.abort.pending.uploads</name>
<value>true</value>
<description>
Should the staging committers abort all pending uploads to the destination
directory
Changing this if more than one partitioned committer is
writing to the same destination tree simultaneously; otherwise
the first job to complete will cancel all outstanding uploads from the
others. However, it may lead to leaked outstanding uploads from failed
tasks. If disabled, configure the bucket lifecycle to remove uploads
after a time period, and/or set up a workflow to explicitly delete
entries. Otherwise there is a risk that uncommitted uploads may run up
bills.
</description>
</property>
<property>
<name>fs.AbstractFileSystem.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3A</value>
<description>The implementation class of the S3A AbstractFileSystem.</description>
</property>
<property>
<name>fs.s3a.list.version</name>
<value>2</value>
<description>
Select which version of the S3 SDK's List Objects API to use. Currently
support 2 (default) and 1 (older API).
</description>
</property>
<property>
<name>fs.s3a.etag.checksum.enabled</name>
<value>false</value>
<description>
Should calls to getFileChecksum() return the etag value of the remote
object.
WARNING: if enabled, distcp operations between HDFS and S3 will fail unless
-skipcrccheck is set.
</description>
</property>
<!-- Azure file system properties -->
<property>
<name>fs.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
<description>The implementation class of the Native Azure Filesystem</description>
</property>
<property>
<name>fs.wasbs.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure</value>
<description>The implementation class of the Secure Native Azure Filesystem</description>
</property>
<property>
<name>fs.azure.secure.mode</name>
<value>false</value>
<description>
Config flag to identify the mode in which fs.azure.NativeAzureFileSystem needs
to run under. Setting it "true" would make fs.azure.NativeAzureFileSystem use
SAS keys to communicate with Azure storage.
</description>
</property>
<property>
<name>fs.azure.local.sas.key.mode</name>
<value>false</value>
<description>
Works in conjuction with fs.azure.secure.mode. Setting this config to true
results in fs.azure.NativeAzureFileSystem using the local SAS key generation
where the SAS keys are generating in the same process as fs.azure.NativeAzureFileSystem.
If fs.azure.secure.mode flag is set to false, this flag has no effect.
</description>
</property>
<property>
<name>fs.azure.sas.expiry.period</name>
<value>90d</value>
<description>
The default value to be used for expiration period for SAS keys generated.
Can use the following suffix (case insensitive):
ms(millis), s(sec), m(min), h(hour), d(day)
to specify the time (such as 2s, 2m, 1h, etc.).
</description>
</property>
<property>
<name>fs.azure.authorization</name>
<value>false</value>
<description>
Config flag to enable authorization support in WASB. Setting it to "true" enables
authorization support to WASB. Currently WASB authorization requires a remote service
to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
configuration
</description>
</property>
<property>
<name>fs.azure.authorization.caching.enable</name>
<value>true</value>
<description>
Config flag to enable caching of authorization results and saskeys in WASB.
This flag is relevant only when fs.azure.authorization is enabled.
</description>
</property>
<property>
<name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
<value>true</value>
<description>
Use container saskey for access to all blobs within the container.
Blob-specific saskeys are not used when this setting is enabled.
This setting provides better performance compared to blob-specific saskeys.
</description>
</property>
<property>
<name>io.seqfile.compress.blocksize</name>
<value>1000000</value>
<description>The minimum block size for compression in block compressed
SequenceFiles.
</description>
</property>
<property>
<name>io.mapfile.bloom.size</name>
<value>1048576</value>
<description>The size of BloomFilter-s used in BloomMapFile. Each time this many
keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
Larger values minimize the number of filters, which slightly increases the performance,
but may waste too much space if the total number of keys is usually much smaller
than this number.
</description>
</property>
<property>
<name>io.mapfile.bloom.error.rate</name>
<value>0.005</value>
<description>The rate of false positives in BloomFilter-s used in BloomMapFile.
As this value decreases, the size of BloomFilter-s increases exponentially. This
value is the probability of encountering false positives (default is 0.5%).
</description>
</property>
<property>
<name>hadoop.util.hash.type</name>
<value>murmur</value>
<description>The default implementation of Hash. Currently this can take one of the
two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
</description>
</property>
<!-- ipc properties -->
<property>
<name>ipc.client.idlethreshold</name>
<value>4000</value>
<description>Defines the threshold number of connections after which
connections will be inspected for idleness.
</description>
</property>
<property>
<name>ipc.client.kill.max</name>
<value>10</value>
<description>Defines the maximum number of clients to disconnect in one go.
</description>
</property>
<property>
<name>ipc.client.connection.maxidletime</name>
<value>10000</value>
<description>The maximum time in msec after which a client will bring down the
connection to the server.
</description>
</property>
<property>
<name>ipc.client.connect.max.retries</name>
<value>10</value>
<description>Indicates the number of retries a client will make to establish
a server connection.
</description>
</property>
<property>
<name>ipc.client.connect.retry.interval</name>
<value>1000</value>
<description>Indicates the number of milliseconds a client will wait for
before retrying to establish a server connection.
</description>
</property>
<property>
<name>ipc.client.connect.timeout</name>
<value>20000</value>
<description>Indicates the number of milliseconds a client will wait for the
socket to establish a server connection.
</description>
</property>
<property>
<name>ipc.client.connect.max.retries.on.timeouts</name>
<value>45</value>
<description>Indicates the number of retries a client will make on socket timeout
to establish a server connection.
</description>
</property>
<property>
<name>ipc.client.tcpnodelay</name>
<value>true</value>
<description>Use TCP_NODELAY flag to bypass Nagle's algorithm transmission delays.
</description>
</property>
<property>
<name>ipc.client.low-latency</name>
<value>false</value>
<description>Use low-latency QoS markers for IPC connections.
</description>
</property>
<property>
<name>ipc.client.ping</name>
<value>true</value>
<description>Send a ping to the server when timeout on reading the response,
if set to true. If no failure is detected, the client retries until at least
a byte is read or the time given by ipc.client.rpc-timeout.ms is passed.
</description>
</property>
<property>
<name>ipc.ping.interval</name>
<value>60000</value>
<description>Timeout on waiting response from server, in milliseconds.
The client will send ping when the interval is passed without receiving bytes,
if ipc.client.ping is set to true.
</description>
</property>
<property>
<name>ipc.client.rpc-timeout.ms</name>
<value>0</value>
<description>Timeout on waiting response from server, in milliseconds.
If ipc.client.ping is set to true and this rpc-timeout is greater than
the value of ipc.ping.interval, the effective value of the rpc-timeout is
rounded up to multiple of ipc.ping.interval.
</description>
</property>
<property>
<name>ipc.server.listen.queue.size</name>
<value>128</value>
<description>Indicates the length of the listen queue for servers accepting
client connections.
</description>
</property>
<property>
<name>ipc.server.log.slow.rpc</name>
<value>false</value>
<description>This setting is useful to troubleshoot performance issues for
various services. If this value is set to true then we log requests that
fall into 99th percentile as well as increment RpcSlowCalls counter.
</description>
</property>
<property>
<name>ipc.maximum.data.length</name>
<value>67108864</value>
<description>This indicates the maximum IPC message length (bytes) that can be
accepted by the server. Messages larger than this value are rejected by the
immediately to avoid possible OOMs. This setting should rarely need to be
changed.
</description>
</property>
<property>
<name>ipc.maximum.response.length</name>
<value>134217728</value>
<description>This indicates the maximum IPC message length (bytes) that can be
accepted by the client. Messages larger than this value are rejected
immediately to avoid possible OOMs. This setting should rarely need to be
changed. Set to 0 to disable.
</description>
</property>
<!-- Proxy Configuration -->
<property>
<name>hadoop.security.impersonation.provider.class</name>
<value></value>
<description>A class which implements ImpersonationProvider interface, used to
authorize whether one user can impersonate a specific user.
If not specified, the DefaultImpersonationProvider will be used.
If a class is specified, then that class will be used to determine
the impersonation capability.
</description>
</property>
<property>
<name>hadoop.rpc.socket.factory.class.default</name>
<value>org.apache.hadoop.net.StandardSocketFactory</value>
<description> Default SocketFactory to use. This parameter is expected to be
formatted as "package.FactoryClassName".
</description>
</property>
<property>
<name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
<value></value>
<description> SocketFactory to use to connect to a DFS. If null or empty, use
hadoop.rpc.socket.class.default. This socket factory is also used by
DFSClient to create sockets to DataNodes.
</description>
</property>
<property>
<name>hadoop.socks.server</name>
<value></value>
<description> Address (host:port) of the SOCKS server to be used by the
SocksSocketFactory.
</description>
</property>
<!-- Topology Configuration -->
<property>
<name>net.topology.node.switch.mapping.impl</name>
<value>org.apache.hadoop.net.ScriptBasedMapping</value>
<description> The default implementation of the DNSToSwitchMapping. It
invokes a script specified in net.topology.script.file.name to resolve
node names. If the value for net.topology.script.file.name is not set, the
default value of DEFAULT_RACK is returned for all node names.
</description>
</property>
<property>
<name>net.topology.impl</name>
<value>org.apache.hadoop.net.NetworkTopology</value>
<description> The default implementation of NetworkTopology which is classic three layer one.
</description>
</property>
<property>
<name>net.topology.script.file.name</name>
<value></value>
<description> The script name that should be invoked to resolve DNS names to
NetworkTopology names. Example: the script would take host.foo.bar as an
argument, and return /rack1 as the output.
</description>
</property>
<property>
<name>net.topology.script.number.args</name>
<value>100</value>
<description> The max number of args that the script configured with
net.topology.script.file.name should be run with. Each arg is an
IP address.
</description>
</property>
<property>
<name>net.topology.table.file.name</name>
<value></value>
<description> The file name for a topology file, which is used when the
net.topology.node.switch.mapping.impl property is set to
org.apache.hadoop.net.TableMapping. The file format is a two column text
file, with columns separated by whitespace. The first column is a DNS or
IP address and the second column specifies the rack where the address maps.
If no entry corresponding to a host in the cluster is found, then
/default-rack is assumed.
</description>
</property>
<!-- Local file system -->
<property>
<name>file.stream-buffer-size</name>
<value>4096</value>
<description>The size of buffer to stream files.
The size of this buffer should probably be a multiple of hardware
page size (4096 on Intel x86), and it determines how much data is
buffered during read and write operations.</description>
</property>
<property>
<name>file.bytes-per-checksum</name>
<value>512</value>
<description>The number of bytes per checksum. Must not be larger than
file.stream-buffer-size</description>
</property>
<property>
<name>file.client-write-packet-size</name>
<value>65536</value>
<description>Packet size for clients to write</description>
</property>
<property>
<name>file.blocksize</name>
<value>67108864</value>
<description>Block size</description>
</property>
<property>
<name>file.replication</name>
<value>1</value>
<description>Replication factor</description>
</property>
<!-- FTP file system -->
<property>
<name>ftp.stream-buffer-size</name>
<value>4096</value>
<description>The size of buffer to stream files.
The size of this buffer should probably be a multiple of hardware
page size (4096 on Intel x86), and it determines how much data is
buffered during read and write operations.</description>
</property>
<property>
<name>ftp.bytes-per-checksum</name>
<value>512</value>
<description>The number of bytes per checksum. Must not be larger than
ftp.stream-buffer-size</description>
</property>
<property>
<name>ftp.client-write-packet-size</name>
<value>65536</value>
<description>Packet size for clients to write</description>
</property>
<property>
<name>ftp.blocksize</name>
<value>67108864</value>
<description>Block size</description>
</property>
<property>
<name>ftp.replication</name>
<value>3</value>
<description>Replication factor</description>
</property>
<!-- Tfile -->
<property>
<name>tfile.io.chunk.size</name>
<value>1048576</value>
<description>
Value chunk size in bytes. Default to
1MB. Values of the length less than the chunk size is
guaranteed to have known value length in read time (See also
TFile.Reader.Scanner.Entry.isValueLengthKnown()).
</description>
</property>
<property>
<name>tfile.fs.output.buffer.size</name>
<value>262144</value>
<description>
Buffer size used for FSDataOutputStream in bytes.
</description>
</property>
<property>
<name>tfile.fs.input.buffer.size</name>
<value>262144</value>
<description>
Buffer size used for FSDataInputStream in bytes.
</description>
</property>
<!-- HTTP web-consoles Authentication -->
<property>
<name>hadoop.http.authentication.type</name>
<value>simple</value>
<description>
Defines authentication used for Oozie HTTP endpoint.
Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
</description>
</property>
<property>
<name>hadoop.http.authentication.token.validity</name>
<value>36000</value>
<description>
Indicates how long (in seconds) an authentication token is valid before it has
to be renewed.
</description>
</property>
<property>
<name>hadoop.http.authentication.signature.secret.file</name>
<value>${user.home}/hadoop-http-auth-signature-secret</value>
<description>
The signature secret for signing the authentication tokens.
The same secret should be used for JT/NN/DN/TT configurations.
</description>
</property>
<property>
<name>hadoop.http.authentication.cookie.domain</name>
<value></value>
<description>
The domain to use for the HTTP cookie that stores the authentication token.
In order to authentiation to work correctly across all Hadoop nodes web-consoles
the domain must be correctly set.
IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings.
For this setting to work properly all nodes in the cluster must be configured
to generate URLs with hostname.domain names on it.
</description>
</property>
<property>
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
<value>true</value>
<description>
Indicates if anonymous requests are allowed when using 'simple' authentication.
</description>
</property>
<property>
<name>hadoop.http.authentication.kerberos.principal</name>
<value>HTTP/_HOST@LOCALHOST</value>
<description>
Indicates the Kerberos principal to be used for HTTP endpoint.
The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.
</description>
</property>
<property>
<name>hadoop.http.authentication.kerberos.keytab</name>
<value>${user.home}/hadoop.keytab</value>
<description>
Location of the keytab file with the credentials for the principal.
Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop.
</description>
</property>
<!-- HTTP CORS support -->
<property>
<name>hadoop.http.cross-origin.enabled</name>
<value>false</value>
<description>Enable/disable the cross-origin (CORS) filter.</description>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-origins</name>
<value>*</value>
<description>Comma separated list of origins that are allowed for web services
needing cross-origin (CORS) support. If a value in the list contains an
asterix (*), a regex pattern, escaping any dots ('.' -> '\.') and replacing
the asterix such that it captures any characters ('*' -> '.*'), is generated.
Values prefixed with 'regex:' are interpreted directly as regular expressions,
e.g. use the expression 'regex:https:\/\/foo\.bar:([0-9]+)' to allow any
origin using the 'http' or 'https' protocol in the domain 'foo.bar' on any
port. The use of simple wildcards ('*') is discouraged, and only available for
backward compatibility.</description>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-methods</name>
<value>GET,POST,HEAD</value>
<description>Comma separated list of methods that are allowed for web
services needing cross-origin (CORS) support.</description>
</property>
<property>
<name>hadoop.http.cross-origin.allowed-headers</name>
<value>X-Requested-With,Content-Type,Accept,Origin</value>
<description>Comma separated list of headers that are allowed for web
services needing cross-origin (CORS) support.</description>
</property>
<property>
<name>hadoop.http.cross-origin.max-age</name>
<value>1800</value>
<description>The number of seconds a pre-flighted request can be cached
for web services needing cross-origin (CORS) support.</description>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value></value>
<description>
List of fencing methods to use for service fencing. May contain
builtin methods (eg shell and sshfence) or user-defined method.
</description>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
<description>
SSH connection timeout, in milliseconds, to use with the builtin
sshfence fencer.
</description>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value></value>
<description>
The SSH private key files to use with the builtin sshfence fencer.
</description>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<description>
A list of ZooKeeper server addresses, separated by commas, that are
to be used by the ZKFailoverController in automatic failover.
</description>
</property>
<property>
<name>ha.zookeeper.session-timeout.ms</name>
<value>10000</value>
<description>
The session timeout to use when the ZKFC connects to ZooKeeper.
Setting this value to a lower value implies that server crashes
will be detected more quickly, but risks triggering failover too
aggressively in the case of a transient error or network blip.
</description>
</property>
<property>
<name>ha.zookeeper.parent-znode</name>
<value>/hadoop-ha</value>
<description>
The ZooKeeper znode under which the ZK failover controller stores
its information. Note that the nameservice ID is automatically
appended to this znode, so it is not normally necessary to
configure this, even in a federated environment.
</description>
</property>
<property>
<name>ha.zookeeper.acl</name>
<value>world:anyone:rwcda</value>
<description>
A comma-separated list of ZooKeeper ACLs to apply to the znodes
used by automatic failover. These ACLs are specified in the same
format as used by the ZooKeeper CLI.
If the ACL itself contains secrets, you may instead specify a
path to a file, prefixed with the '@' symbol, and the value of
this configuration will be loaded from within.
</description>
</property>
<property>
<name>ha.zookeeper.auth</name>
<value></value>
<description>
A comma-separated list of ZooKeeper authentications to add when
connecting to ZooKeeper. These are specified in the same format
as used by the "addauth" command in the ZK CLI. It is
important that the authentications specified here are sufficient
to access znodes with the ACL specified in ha.zookeeper.acl.
If the auths contain secrets, you may instead specify a
path to a file, prefixed with the '@' symbol, and the value of
this configuration will be loaded from within.
</description>
</property>
<!-- Static Web User Filter properties. -->
<property>
<name>hadoop.http.staticuser.user</name>
<value>dr.who</value>
<description>
The user name to filter as, on static web filters
while rendering content. An example use is the HDFS
web UI (user to be used for browsing files).
</description>
</property>
<!-- SSLFactory configuration -->
<property>
<name>hadoop.ssl.keystores.factory.class</name>
<value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
<description>
The keystores factory to use for retrieving certificates.
</description>
</property>
<property>
<name>hadoop.ssl.require.client.cert</name>
<value>false</value>
<description>Whether client certificates are required</description>
</property>
<property>
<name>hadoop.ssl.hostname.verifier</name>
<value>DEFAULT</value>
<description>
The hostname verifier to provide for HttpsURLConnections.
Valid values are: DEFAULT, STRICT, STRICT_IE6, DEFAULT_AND_LOCALHOST and
ALLOW_ALL
</description>
</property>
<property>
<name>hadoop.ssl.server.conf</name>
<value>ssl-server.xml</value>
<description>
Resource file from which ssl server keystore information will be extracted.
This file is looked up in the classpath, typically it should be in Hadoop
conf/ directory.
</description>
</property>
<property>
<name>hadoop.ssl.client.conf</name>
<value>ssl-client.xml</value>
<description>
Resource file from which ssl client keystore information will be extracted
This file is looked up in the classpath, typically it should be in Hadoop
conf/ directory.
</description>
</property>
<property>
<name>hadoop.ssl.enabled</name>
<value>false</value>
<description>
Deprecated. Use dfs.http.policy and yarn.http.policy instead.
</description>
</property>
<property>
<name>hadoop.ssl.enabled.protocols</name>
<value>TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2</value>
<description>
The supported SSL protocols.
</description>
</property>
<property>
<name>hadoop.jetty.logs.serve.aliases</name>
<value>true</value>
<description>
Enable/Disable aliases serving from jetty
</description>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>022</value>
<description>
The umask used when creating files and directories.
Can be in octal or in symbolic. Examples are:
"022" (octal for u=rwx,g=r-x,o=r-x in symbolic),
or "u=rwx,g=rwx,o=" (symbolic for 007 in octal).
</description>
</property>
<!-- ha properties -->
<property>
<name>ha.health-monitor.connect-retry-interval.ms</name>
<value>1000</value>
<description>
How often to retry connecting to the service.
</description>
</property>
<property>
<name>ha.health-monitor.check-interval.ms</name>
<value>1000</value>
<description>
How often to check the service.
</description>
</property>
<property>
<name>ha.health-monitor.sleep-after-disconnect.ms</name>
<value>1000</value>
<description>
How long to sleep after an unexpected RPC error.
</description>
</property>
<property>
<name>ha.health-monitor.rpc-timeout.ms</name>
<value>45000</value>
<description>
Timeout for the actual monitorHealth() calls.
</description>
</property>
<property>
<name>ha.failover-controller.new-active.rpc-timeout.ms</name>
<value>60000</value>
<description>
Timeout that the FC waits for the new active to become active
</description>
</property>
<property>
<name>ha.failover-controller.graceful-fence.rpc-timeout.ms</name>
<value>5000</value>
<description>
Timeout that the FC waits for the old active to go to standby
</description>
</property>
<property>
<name>ha.failover-controller.graceful-fence.connection.retries</name>
<value>1</value>
<description>
FC connection retries for graceful fencing
</description>
</property>
<property>
<name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
<value>20000</value>
<description>
Timeout that the CLI (manual) FC waits for monitorHealth, getServiceState
</description>
</property>
<property>
<name>ipc.client.fallback-to-simple-auth-allowed</name>
<value>false</value>
<description>
When a client is configured to attempt a secure connection, but attempts to
connect to an insecure server, that server may instruct the client to
switch to SASL SIMPLE (unsecure) authentication. This setting controls
whether or not the client will accept this instruction from the server.
When false (the default), the client will not allow the fallback to SIMPLE
authentication, and will abort the connection.
</description>
</property>
<property>
<name>fs.client.resolve.remote.symlinks</name>
<value>true</value>
<description>
Whether to resolve symlinks when accessing a remote Hadoop filesystem.
Setting this to false causes an exception to be thrown upon encountering
a symlink. This setting does not apply to local filesystems, which
automatically resolve local symlinks.
</description>
</property>
<property>
<name>nfs.exports.allowed.hosts</name>
<value>* rw</value>
<description>
By default, the export can be mounted by any client. The value string
contains machine name and access privilege, separated by whitespace
characters. The machine name format can be a single host, a Java regular
expression, or an IPv4 address. The access privilege uses rw or ro to
specify read/write or read-only access of the machines to exports. If the
access privilege is not provided, the default is read-only. Entries are separated by ";".
For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;".
Only the NFS gateway needs to restart after this property is updated.
</description>
</property>
<property>
<name>hadoop.user.group.static.mapping.overrides</name>
<value>dr.who=;</value>
<description>
Static mapping of user to groups. This will override the groups if
available in the system for the specified user. In other words, groups
look-up will not happen for these users, instead groups mapped in this
configuration will be used.
Mapping should be in this format.
user1=group1,group2;user2=;user3=group2;
Default, "dr.who=;" will consider "dr.who" as user without groups.
</description>
</property>
<property>
<name>rpc.metrics.quantile.enable</name>
<value>false</value>
<description>
Setting this property to true and rpc.metrics.percentiles.intervals
to a comma-separated list of the granularity in seconds, the
50/75/90/95/99th percentile latency for rpc queue/processing time in
milliseconds are added to rpc metrics.
</description>
</property>
<property>
<name>rpc.metrics.percentiles.intervals</name>
<value></value>
<description>
A comma-separated list of the granularity in seconds for the metrics which
describe the 50/75/90/95/99th percentile latency for rpc queue/processing
time. The metrics are outputted if rpc.metrics.quantile.enable is set to
true.
</description>
</property>
<property>
<name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name>
<value></value>
<description>
The prefix for a given crypto codec, contains a comma-separated
list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE).
The first implementation will be used if available, others are fallbacks.
</description>
</property>
<property>
<name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
<value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
<description>
Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
The first implementation will be used if available, others are fallbacks.
</description>
</property>
<property>
<name>hadoop.security.crypto.cipher.suite</name>
<value>AES/CTR/NoPadding</value>
<description>
Cipher suite for crypto codec.
</description>
</property>
<property>
<name>hadoop.security.crypto.jce.provider</name>
<value></value>
<description>
The JCE provider name used in CryptoCodec.
</description>
</property>
<property>
<name>hadoop.security.crypto.jceks.key.serialfilter</name>
<description>
Enhanced KeyStore Mechanisms in JDK 8u171 introduced jceks.key.serialFilter.
If jceks.key.serialFilter is configured, the JCEKS KeyStore uses it during
the deserialization of the encrypted Key object stored inside a
SecretKeyEntry.
If jceks.key.serialFilter is not configured it will cause an error when
recovering keystore file in KeyProviderFactory when recovering key from
keystore file using JDK 8u171 or newer. The filter pattern uses the same
format as jdk.serialFilter.
The value of this property will be used as the following:
1. The value of jceks.key.serialFilter system property takes precedence
over the value of this property.
2. In the absence of jceks.key.serialFilter system property the value of
this property will be set as the value of jceks.key.serialFilter.
3. If the value of this property and jceks.key.serialFilter system
property has not been set, org.apache.hadoop.crypto.key.KeyProvider
sets a default value for jceks.key.serialFilter.
</description>
</property>
<property>
<name>hadoop.security.crypto.buffer.size</name>
<value>8192</value>
<description>
The buffer size used by CryptoInputStream and CryptoOutputStream.
</description>
</property>
<property>
<name>hadoop.security.java.secure.random.algorithm</name>
<value>SHA1PRNG</value>
<description>
The java secure random algorithm.
</description>
</property>
<property>
<name>hadoop.security.secure.random.impl</name>
<value></value>
<description>
Implementation of secure random.
</description>
</property>
<property>
<name>hadoop.security.random.device.file.path</name>
<value>/dev/urandom</value>
<description>
OS security random device file path.
</description>
</property>
<property>
<name>hadoop.security.key.provider.path</name>
<description>
The KeyProvider to use when managing zone keys, and interacting with
encryption keys when reading and writing to an encryption zone.
For hdfs clients, the provider path will be same as namenode's
provider path.
</description>
</property>
<property>
<name>hadoop.security.key.default.bitlength</name>
<value>128</value>
<description>
The length (bits) of keys we want the KeyProvider to produce. Key length
defines the upper-bound on an algorithm's security, ideally, it would
coincide with the lower-bound on an algorithm's security.
</description>
</property>
<property>
<name>hadoop.security.key.default.cipher</name>
<value>AES/CTR/NoPadding</value>
<description>
This indicates the algorithm that be used by KeyProvider for generating
key, and will be converted to CipherSuite when creating encryption zone.
</description>
</property>
<property>
<name>fs.har.impl.disable.cache</name>
<value>true</value>
<description>Don't cache 'har' filesystem instances.</description>
</property>
<!--- KMSClientProvider configurations -->
<property>
<name>hadoop.security.kms.client.authentication.retry-count</name>
<value>1</value>
<description>
Number of time to retry connecting to KMS on authentication failure
</description>
</property>
<property>
<name>hadoop.security.kms.client.encrypted.key.cache.size</name>
<value>500</value>
<description>
Size of the EncryptedKeyVersion cache Queue for each key
</description>
</property>
<property>
<name>hadoop.security.kms.client.encrypted.key.cache.low-watermark</name>
<value>0.3f</value>
<description>
If size of the EncryptedKeyVersion cache Queue falls below the
low watermark, this cache queue will be scheduled for a refill
</description>
</property>
<property>
<name>hadoop.security.kms.client.encrypted.key.cache.num.refill.threads</name>
<value>2</value>
<description>
Number of threads to use for refilling depleted EncryptedKeyVersion
cache Queues
</description>
</property>
<property>
<name>hadoop.security.kms.client.encrypted.key.cache.expiry</name>
<value>43200000</value>
<description>
Cache expiry time for a Key, after which the cache Queue for this
key will be dropped. Default = 12hrs
</description>
</property>
<property>
<name>hadoop.security.kms.client.timeout</name>
<value>60</value>
<description>
Sets value for KMS client connection timeout, and the read timeout
to KMS servers.
</description>
</property>
<property>
<name>hadoop.security.kms.client.failover.sleep.base.millis</name>
<value>100</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the base value used in the failover calculation. The
first failover will retry immediately. The 2nd failover attempt
will delay at least hadoop.security.client.failover.sleep.base.millis
milliseconds. And so on.
</description>
</property>
<property>
<name>hadoop.security.kms.client.failover.sleep.max.millis</name>
<value>2000</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the maximum value to wait between failovers.
Specifically, the time between two failover attempts will not
exceed +/- 50% of hadoop.security.client.failover.sleep.max.millis
milliseconds.
</description>
</property>
<property>
<name>ipc.server.max.connections</name>
<value>0</value>
<description>The maximum number of concurrent connections a server is allowed
to accept. If this limit is exceeded, incoming connections will first fill
the listen queue and then may go to an OS-specific listen overflow queue.
The client may fail or timeout, but the server can avoid running out of file
descriptors using this feature. 0 means no limit.
</description>
</property>
<!-- YARN registry -->
<property>
<name>hadoop.registry.rm.enabled</name>
<value>false</value>
<description>
Is the registry enabled in the YARN Resource Manager
If true, the YARN RM will, as needed.
create the user and system paths, and purge
service records when containers, application attempts
and applications complete.
If false, the paths must be created by other means,
and no automatic cleanup of service records will take place.
</description>
</property>
<property>
<name>hadoop.registry.zk.root</name>
<value>/registry</value>
<description>
The root zookeeper node for the registry
</description>
</property>
<property>
<name>hadoop.registry.zk.session.timeout.ms</name>
<value>60000</value>
<description>
Zookeeper session timeout in milliseconds
</description>
</property>
<property>
<name>hadoop.registry.zk.connection.timeout.ms</name>
<value>15000</value>
<description>
Zookeeper connection timeout in milliseconds
</description>
</property>
<property>
<name>hadoop.registry.zk.retry.times</name>
<value>5</value>
<description>
Zookeeper connection retry count before failing
</description>
</property>
<property>
<name>hadoop.registry.zk.retry.interval.ms</name>
<value>1000</value>
<description>
</description>
</property>
<property>
<name>hadoop.registry.zk.retry.ceiling.ms</name>
<value>60000</value>
<description>
Zookeeper retry limit in milliseconds, during
exponential backoff.
This places a limit even
if the retry times and interval limit, combined
with the backoff policy, result in a long retry
period
</description>
</property>
<property>
<name>hadoop.registry.zk.quorum</name>
<value>localhost:2181</value>
<description>
List of hostname:port pairs defining the
zookeeper quorum binding for the registry
</description>
</property>
<property>
<name>hadoop.registry.secure</name>
<value>false</value>
<description>
Key to set if the registry is secure. Turning it on
changes the permissions policy from "open access"
to restrictions on kerberos with the option of
a user adding one or more auth key pairs down their
own tree.
</description>
</property>
<property>
<name>hadoop.registry.system.acls</name>
<value>sasl:yarn@, sasl:mapred@, sasl:hdfs@</value>
<description>
A comma separated list of Zookeeper ACL identifiers with
system access to the registry in a secure cluster.
These are given full access to all entries.
If there is an "@" at the end of a SASL entry it
instructs the registry client to append the default kerberos domain.
</description>
</property>
<property>
<name>hadoop.registry.kerberos.realm</name>
<value></value>
<description>
The kerberos realm: used to set the realm of
system principals which do not declare their realm,
and any other accounts that need the value.
If empty, the default realm of the running process
is used.
If neither are known and the realm is needed, then the registry
service/client will fail.
</description>
</property>
<property>
<name>hadoop.registry.jaas.context</name>
<value>Client</value>
<description>
Key to define the JAAS context. Used in secure
mode
</description>
</property>
<property>
<name>hadoop.shell.missing.defaultFs.warning</name>
<value>false</value>
<description>
Enable hdfs shell commands to display warnings if (fs.defaultFS) property
is not set.
</description>
</property>
<property>
<name>hadoop.shell.safely.delete.limit.num.files</name>
<value>100</value>
<description>Used by -safely option of hadoop fs shell -rm command to avoid
accidental deletion of large directories. When enabled, the -rm command
requires confirmation if the number of files to be deleted is greater than
this limit. The default limit is 100 files. The warning is disabled if
the limit is 0 or the -safely is not specified in -rm command.
</description>
</property>
<property>
<name>fs.client.htrace.sampler.classes</name>
<value></value>
<description>The class names of the HTrace Samplers to use for Hadoop
filesystem clients.
</description>
</property>
<property>
<name>hadoop.htrace.span.receiver.classes</name>
<value></value>
<description>The class names of the Span Receivers to use for Hadoop.
</description>
</property>
<property>
<name>hadoop.http.logs.enabled</name>
<value>true</value>
<description>
Enable the "/logs" endpoint on all Hadoop daemons, which serves local
logs, but may be considered a security risk due to it listing the contents
of a directory.
</description>
</property>
<property>
<name>fs.client.resolve.topology.enabled</name>
<value>false</value>
<description>Whether the client machine will use the class specified by
property net.topology.node.switch.mapping.impl to compute the network
distance between itself and remote machines of the FileSystem. Additional
properties might need to be configured depending on the class specified
in net.topology.node.switch.mapping.impl. For example, if
org.apache.hadoop.net.ScriptBasedMapping is used, a valid script file
needs to be specified in net.topology.script.file.name.
</description>
</property>
<!-- Azure Data Lake File System Configurations -->
<property>
<name>fs.adl.impl</name>
<value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
</property>
<property>
<name>fs.AbstractFileSystem.adl.impl</name>
<value>org.apache.hadoop.fs.adl.Adl</value>
</property>
<property>
<name>adl.feature.ownerandgroup.enableupn</name>
<value>false</value>
<description>
When true : User and Group in FileStatus/AclStatus response is
represented as user friendly name as per Azure AD profile.
When false (default) : User and Group in FileStatus/AclStatus
response is represented by the unique identifier from Azure AD
profile (Object ID as GUID).
For optimal performance, false is recommended.
</description>
</property>
<property>
<name>fs.adl.oauth2.access.token.provider.type</name>
<value>ClientCredential</value>
<description>
Defines Azure Active Directory OAuth2 access token provider type.
Supported types are ClientCredential, RefreshToken, MSI, DeviceCode,
and Custom.
The ClientCredential type requires property fs.adl.oauth2.client.id,
fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
The RefreshToken type requires property fs.adl.oauth2.client.id and
fs.adl.oauth2.refresh.token.
The MSI type reads optional property fs.adl.oauth2.msi.port, if specified.
The DeviceCode type requires property
fs.adl.oauth2.devicecode.clientapp.id.
The Custom type requires property fs.adl.oauth2.access.token.provider.
</description>
</property>
<property>
<name>fs.adl.oauth2.client.id</name>
<value></value>
<description>The OAuth2 client id.</description>
</property>
<property>
<name>fs.adl.oauth2.credential</name>
<value></value>
<description>The OAuth2 access key.</description>
</property>
<property>
<name>fs.adl.oauth2.refresh.url</name>
<value></value>
<description>The OAuth2 token endpoint.</description>
</property>
<property>
<name>fs.adl.oauth2.refresh.token</name>
<value></value>
<description>The OAuth2 refresh token.</description>
</property>
<property>
<name>fs.adl.oauth2.access.token.provider</name>
<value></value>
<description>
The class name of the OAuth2 access token provider.
</description>
</property>
<property>
<name>fs.adl.oauth2.msi.port</name>
<value></value>
<description>
The localhost port for the MSI token service. This is the port specified
when creating the Azure VM. The default, if this setting is not specified,
is 50342.
Used by MSI token provider.
</description>
</property>
<property>
<name>fs.adl.oauth2.devicecode.clientapp.id</name>
<value></value>
<description>
The app id of the AAD native app in whose context the auth request
should be made.
Used by DeviceCode token provider.
</description>
</property>
<!-- Azure Data Lake File System Configurations Ends Here-->
<property>
<name>hadoop.caller.context.enabled</name>
<value>false</value>
<description>When the feature is enabled, additional fields are written into
name-node audit log records for auditing coarse granularity operations.
</description>
</property>
<property>
<name>hadoop.caller.context.max.size</name>
<value>128</value>
<description>The maximum bytes a caller context string can have. If the
passed caller context is longer than this maximum bytes, client will
truncate it before sending to server. Note that the server may have a
different maximum size, and will truncate the caller context to the
maximum size it allows.
</description>
</property>
<property>
<name>hadoop.caller.context.signature.max.size</name>
<value>40</value>
<description>
The caller's signature (optional) is for offline validation. If the
signature exceeds the maximum allowed bytes in server, the caller context
will be abandoned, in which case the caller context will not be recorded
in audit logs.
</description>
</property>
<!-- SequenceFile's Sorter properties -->
<property>
<name>seq.io.sort.mb</name>
<value>100</value>
<description>
The total amount of buffer memory to use while sorting files,
while using SequenceFile.Sorter, in megabytes. By default,
gives each merge stream 1MB, which should minimize seeks.
</description>
</property>
<property>
<name>seq.io.sort.factor</name>
<value>100</value>
<description>
The number of streams to merge at once while sorting
files using SequenceFile.Sorter.
This determines the number of open file handles.
</description>
</property>
<property>
<name>hadoop.zk.address</name>
<!--value>127.0.0.1:2181</value-->
<description>Host:Port of the ZooKeeper server to be used.
</description>
</property>
<property>
<name>hadoop.zk.num-retries</name>
<value>1000</value>
<description>Number of tries to connect to ZooKeeper.</description>
</property>
<property>
<name>hadoop.zk.retry-interval-ms</name>
<value>1000</value>
<description>Retry interval in milliseconds when connecting to ZooKeeper.
</description>
</property>
<property>
<name>hadoop.zk.timeout-ms</name>
<value>10000</value>
<description>ZooKeeper session timeout in milliseconds. Session expiration
is managed by the ZooKeeper cluster itself, not by the client. This value is
used by the cluster to determine when the client's session expires.
Expirations happens when the cluster does not hear from the client within
the specified session timeout period (i.e. no heartbeat).</description>
</property>
<property>
<name>hadoop.zk.acl</name>
<value>world:anyone:rwcda</value>
<description>ACL's to be used for ZooKeeper znodes.</description>
</property>
<property>
<name>hadoop.zk.auth</name>
<description>
Specify the auths to be used for the ACL's specified in hadoop.zk.acl.
This takes a comma-separated list of authentication mechanisms, each of the
form 'scheme:auth' (the same syntax used for the 'addAuth' command in
the ZK CLI).
</description>
</property>
<property>
<name>hadoop.system.tags</name>
<value>YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT
,SERVER,DEBUG,DEPRICATED,COMMON,OPTIONAL</value>
<description>
System tags to group related properties together.
</description>
</property>
<property>
<name>ipc.client.bind.wildcard.addr</name>
<value>false</value>
<description>When set to true Clients will bind socket to wildcard
address. (i.e 0.0.0.0)
</description>
</property>
</configuration>
2.hdfs-default.xml
<xml version="1.0">
<xml-stylesheet type="text/xsl" href="configuration.xsl">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Do not modify this file directly. Instead, copy entries that you -->
<!-- wish to modify from this file into hdfs-site.xml and change them -->
<!-- there. If hdfs-site.xml does not already exist, create it. -->
<configuration>
<property>
<name>hadoop.hdfs.configuration.version</name>
<value>1</value>
<description>version of this configuration file</description>
</property>
<property>
<name>dfs.namenode.rpc-address</name>
<value></value>
<description>
RPC address that handles all clients requests. In the case of HA/Federation where multiple namenodes exist,
the name service id is added to the name e.g. dfs.namenode.rpc-address.ns1
dfs.namenode.rpc-address.EXAMPLENAMESERVICE
The value of this property will take the form of nn-host1:rpc-port. The NameNode's default RPC port is 8020.
</description>
</property>
<property>
<name>dfs.namenode.rpc-bind-host</name>
<value></value>
<description>
The actual address the RPC server will bind to. If this optional address is
set, it overrides only the hostname portion of dfs.namenode.rpc-address.
It can also be specified per name node or name service for HA/Federation.
This is useful for making the name node listen on all interfaces by
setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.namenode.servicerpc-address</name>
<value></value>
<description>
RPC address for HDFS Services communication. BackupNode, Datanodes and all other services should be
connecting to this address if it is configured. In the case of HA/Federation where multiple namenodes exist,
the name service id is added to the name e.g. dfs.namenode.servicerpc-address.ns1
dfs.namenode.rpc-address.EXAMPLENAMESERVICE
The value of this property will take the form of nn-host1:rpc-port.
If the value of this property is unset the value of dfs.namenode.rpc-address will be used as the default.
</description>
</property>
<property>
<name>dfs.namenode.servicerpc-bind-host</name>
<value></value>
<description>
The actual address the service RPC server will bind to. If this optional address is
set, it overrides only the hostname portion of dfs.namenode.servicerpc-address.
It can also be specified per name node or name service for HA/Federation.
This is useful for making the name node listen on all interfaces by
setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.namenode.lifeline.rpc-address</name>
<value></value>
<description>
NameNode RPC lifeline address. This is an optional separate RPC address
that can be used to isolate health checks and liveness to protect against
resource exhaustion in the main RPC handler pool. In the case of
HA/Federation where multiple NameNodes exist, the name service ID is added
to the name e.g. dfs.namenode.lifeline.rpc-address.ns1. The value of this
property will take the form of nn-host1:rpc-port. If this property is not
defined, then the NameNode will not start a lifeline RPC server. By
default, the property is not defined.
</description>
</property>
<property>
<name>dfs.namenode.lifeline.rpc-bind-host</name>
<value></value>
<description>
The actual address the lifeline RPC server will bind to. If this optional
address is set, it overrides only the hostname portion of
dfs.namenode.lifeline.rpc-address. It can also be specified per name node
or name service for HA/Federation. This is useful for making the name node
listen on all interfaces by setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>0.0.0.0:9868</value>
<description>
The secondary namenode http server address and port.
</description>
</property>
<property>
<name>dfs.namenode.secondary.https-address</name>
<value>0.0.0.0:9869</value>
<description>
The secondary namenode HTTPS server address and port.
</description>
</property>
<property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:9866</value>
<description>
The datanode server address and port for data transfer.
</description>
</property>
<property>
<name>dfs.datanode.http.address</name>
<value>0.0.0.0:9864</value>
<description>
The datanode http server address and port.
</description>
</property>
<property>
<name>dfs.datanode.ipc.address</name>
<value>0.0.0.0:9867</value>
<description>
The datanode ipc server address and port.
</description>
</property>
<property>
<name>dfs.datanode.http.internal-proxy.port</name>
<value>0</value>
<description>
The datanode's internal web proxy port.
By default it selects a random port available in runtime.
</description>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>10</value>
<description>The number of server threads for the datanode.</description>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>0.0.0.0:9870</value>
<description>
The address and the base port where the dfs namenode web ui will listen on.
</description>
</property>
<property>
<name>dfs.namenode.http-bind-host</name>
<value></value>
<description>
The actual address the HTTP server will bind to. If this optional address
is set, it overrides only the hostname portion of dfs.namenode.http-address.
It can also be specified per name node or name service for HA/Federation.
This is useful for making the name node HTTP server listen on all
interfaces by setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.namenode.heartbeat.recheck-interval</name>
<value>300000</value>
<description>
This time decides the interval to check for expired datanodes.
With this value and dfs.heartbeat.interval, the interval of
deciding the datanode is stale or not is also calculated.
The unit of this configuration is millisecond.
</description>
</property>
<property>
<name>dfs.http.policy</name>
<value>HTTP_ONLY</value>
<description>Decide if HTTPS(SSL) is supported on HDFS
This configures the HTTP endpoint for HDFS daemons:
The following values are supported:
- HTTP_ONLY : Service is provided only on http
- HTTPS_ONLY : Service is provided only on https
- HTTP_AND_HTTPS : Service is provided both on http and https
</description>
</property>
<property>
<name>dfs.client.https.need-auth</name>
<value>false</value>
<description>Whether SSL client certificate authentication is required
</description>
</property>
<property>
<name>dfs.client.cached.conn.retry</name>
<value>3</value>
<description>The number of times the HDFS client will pull a socket from the
cache. Once this number is exceeded, the client will try to create a new
socket.
</description>
</property>
<property>
<name>dfs.https.server.keystore.resource</name>
<value>ssl-server.xml</value>
<description>Resource file from which ssl server keystore
information will be extracted
</description>
</property>
<property>
<name>dfs.client.https.keystore.resource</name>
<value>ssl-client.xml</value>
<description>Resource file from which ssl client keystore
information will be extracted
</description>
</property>
<property>
<name>dfs.datanode.https.address</name>
<value>0.0.0.0:9865</value>
<description>The datanode secure http server address and port.</description>
</property>
<property>
<name>dfs.namenode.https-address</name>
<value>0.0.0.0:9871</value>
<description>The namenode secure http server address and port.</description>
</property>
<property>
<name>dfs.namenode.https-bind-host</name>
<value></value>
<description>
The actual address the HTTPS server will bind to. If this optional address
is set, it overrides only the hostname portion of dfs.namenode.https-address.
It can also be specified per name node or name service for HA/Federation.
This is useful for making the name node HTTPS server listen on all
interfaces by setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.datanode.dns.interface</name>
<value>default</value>
<description>
The name of the Network Interface from which a data node should
report its IP address. e.g. eth2. This setting may be required for some
multi-homed nodes where the DataNodes are assigned multiple hostnames
and it is desirable for the DataNodes to use a non-default hostname.
Prefer using hadoop.security.dns.interface over
dfs.datanode.dns.interface.
</description>
</property>
<property>
<name>dfs.datanode.dns.nameserver</name>
<value>default</value>
<description>
The host name or IP address of the name server (DNS) which a DataNode
should use to determine its own host name.
Prefer using hadoop.security.dns.nameserver over
dfs.datanode.dns.nameserver.
</description>
</property>
<property>
<name>dfs.namenode.backup.address</name>
<value>0.0.0.0:50100</value>
<description>
The backup node server address and port.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.namenode.backup.http-address</name>
<value>0.0.0.0:50105</value>
<description>
The backup node http server address and port.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.namenode.redundancy.considerLoad</name>
<value>true</value>
<description>Decide if chooseTarget considers the target's load or not
</description>
</property>
<property>
<name>dfs.namenode.redundancy.considerLoad.factor</name>
<value>2.0</value>
<description>The factor by which a node's load can exceed the average
before being rejected for writes, only if considerLoad is true.
</description>
</property>
<property>
<name>dfs.default.chunk.view.size</name>
<value>32768</value>
<description>The number of bytes to view for a file on the browser.
</description>
</property>
<property>
<name>dfs.datanode.du.reserved.calculator</name>
<value>org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.ReservedSpaceCalculator$ReservedSpaceCalculatorAbsolute</value>
<description>Determines the class of ReservedSpaceCalculator to be used for
calculating disk space reservedfor non-HDFS data. The default calculator is
ReservedSpaceCalculatorAbsolute which will use dfs.datanode.du.reserved
for a static reserved number of bytes. ReservedSpaceCalculatorPercentage
will use dfs.datanode.du.reserved.pct to calculate the reserved number
of bytes based on the size of the storage. ReservedSpaceCalculatorConservative and
ReservedSpaceCalculatorAggressive will use their combination, Conservative will use
maximum, Aggressive minimum. For more details see ReservedSpaceCalculator.
</description>
</property>
<property>
<name>dfs.datanode.du.reserved</name>
<value>0</value>
<description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
Specific storage type based reservation is also supported. The property can be followed with
corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage.
For example, reserved space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved will be used.
</description>
</property>
<property>
<name>dfs.datanode.du.reserved.pct</name>
<value>0</value>
<description>Reserved space in percentage. Read dfs.datanode.du.reserved.calculator to see
when this takes effect. The actual number of bytes reserved will be calculated by using the
total capacity of the data directory in question. Specific storage type based reservation
is also supported. The property can be followed with corresponding storage types
([ssd]/[disk]/[archive]/[ram_disk]) for cluster with heterogeneous storage.
For example, reserved percentage space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.pct.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved.pct will be used.
</description>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/name</value>
<description>Determines where on the local filesystem the DFS name node
should store the name table(fsimage). If this is a comma-delimited list
of directories then the name table is replicated in all of the
directories, for redundancy. </description>
</property>
<property>
<name>dfs.namenode.name.dir.restore</name>
<value>false</value>
<description>Set to true to enable NameNode to attempt recovering a
previously failed dfs.namenode.name.dir. When enabled, a recovery of any
failed directory is attempted during checkpoint.</description>
</property>
<property>
<name>dfs.namenode.fs-limits.max-component-length</name>
<value>255</value>
<description>Defines the maximum number of bytes in UTF-8 encoding in each
component of a path. A value of 0 will disable the check.</description>
</property>
<property>
<name>dfs.namenode.fs-limits.max-directory-items</name>
<value>1048576</value>
<description>Defines the maximum number of items that a directory may
contain. Cannot set the property to a value less than 1 or more than
6400000.</description>
</property>
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>1048576</value>
<description>Minimum block size in bytes, enforced by the Namenode at create
time. This prevents the accidental creation of files with tiny block
sizes (and thus many blocks), which can degrade
performance.</description>
</property>
<property>
<name>dfs.namenode.fs-limits.max-blocks-per-file</name>
<value>10000</value>
<description>Maximum number of blocks per file, enforced by the Namenode on
write. This prevents the creation of extremely large files which can
degrade performance.</description>
</property>
<property>
<name>dfs.namenode.edits.dir</name>
<value>${dfs.namenode.name.dir}</value>
<description>Determines where on the local filesystem the DFS name node
should store the transaction (edits) file. If this is a comma-delimited list
of directories then the transaction file is replicated in all of the
directories, for redundancy. Default value is same as dfs.namenode.name.dir
</description>
</property>
<property>
<name>dfs.namenode.edits.dir.required</name>
<value></value>
<description>This should be a subset of dfs.namenode.edits.dir,
to ensure that the transaction (edits) file
in these places is always up-to-date.
</description>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value></value>
<description>A directory on shared storage between the multiple namenodes
in an HA cluster. This directory will be written by the active and read
by the standby in order to keep the namespaces synchronized. This directory
does not need to be listed in dfs.namenode.edits.dir above. It should be
left empty in a non-HA cluster.
</description>
</property>
<property>
<name>dfs.namenode.edits.journal-plugin.qjournal</name>
<value>org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>true</value>
<description>
If "true", enable permission checking in HDFS.
If "false", permission checking is turned off,
but all other behavior is unchanged.
Switching from one parameter value to the other does not change the mode,
owner or group of files or directories.
</description>
</property>
<property>
<name>dfs.permissions.superusergroup</name>
<value>supergroup</value>
<description>The name of the group of super-users.
The value should be a single group name.
</description>
</property>
<property>
<name>dfs.cluster.administrators</name>
<value></value>
<description>ACL for the admins, this configuration is used to control
who can access the default servlets in the namenode, etc. The value
should be a comma separated list of users and groups. The user list
comes first and is separated by a space followed by the group list,
e.g. "user1,user2 group1,group2". Both users and groups are optional,
so "user1", " group1", "", "user1 group1", "user1,user2 group1,group2"
are all valid (note the leading space in " group1"). '*' grants access
to all users and groups, e.g. '*', '* ' and ' *' are all valid.
</description>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>false</value>
<description>
Set to true to enable support for HDFS ACLs (Access Control Lists). By
default, ACLs are disabled. When ACLs are disabled, the NameNode rejects
all RPCs related to setting or getting ACLs.
</description>
</property>
<property>
<name>dfs.namenode.posix.acl.inheritance.enabled</name>
<value>true</value>
<description>
Set to true to enable POSIX style ACL inheritance. When it is enabled
and the create request comes from a compatible client, the NameNode
will apply default ACLs from the parent directory to the create mode
and ignore the client umask. If no default ACL found, it will apply the
client umask.
</description>
</property>
<property>
<name>dfs.namenode.lazypersist.file.scrub.interval.sec</name>
<value>300</value>
<description>
The NameNode periodically scans the namespace for LazyPersist files with
missing blocks and unlinks them from the namespace. This configuration key
controls the interval between successive scans. If this value is set to 0,
the file scrubber is disabled.
</description>
</property>
<property>
<name>dfs.block.access.token.enable</name>
<value>false</value>
<description>
If "true", access tokens are used as capabilities for accessing datanodes.
If "false", no access tokens are checked on accessing datanodes.
</description>
</property>
<property>
<name>dfs.block.access.key.update.interval</name>
<value>600</value>
<description>
Interval in minutes at which namenode updates its access keys.
</description>
</property>
<property>
<name>dfs.block.access.token.lifetime</name>
<value>600</value>
<description>The lifetime of access tokens in minutes.</description>
</property>
<property>
<name>dfs.block.access.token.protobuf.enable</name>
<value>false</value>
<description>
If "true", block tokens are written using Protocol Buffers.
If "false", block tokens are written using Legacy format.
</description>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
should store its blocks. If this is a comma-delimited
list of directories, then data will be stored in all named
directories, typically on different devices. The directories should be tagged
with corresponding storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS
storage policies. The default storage type will be DISK if the directory does
not have a storage type tagged explicitly. Directories that do not exist will
be created if local filesystem permission allows.
</description>
</property>
<property>
<name>dfs.datanode.data.dir.perm</name>
<value>700</value>
<description>Permissions for the directories on on the local filesystem where
the DFS data node store its blocks. The permissions can either be octal or
symbolic.</description>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
<description>Default block replication.
The actual number of replications can be specified when the file is created.
The default is used if replication is not specified in create time.
</description>
</property>
<property>
<name>dfs.replication.max</name>
<value>512</value>
<description>Maximal block replication.
</description>
</property>
<property>
<name>dfs.namenode.replication.min</name>
<value>1</value>
<description>Minimal block replication.
</description>
</property>
<property>
<name>dfs.namenode.maintenance.replication.min</name>
<value>1</value>
<description>Minimal live block replication in existence of maintenance mode.
</description>
</property>
<property>
<name>dfs.namenode.safemode.replication.min</name>
<value></value>
<description>
a separate minimum replication factor for calculating safe block count.
This is an expert level setting.
Setting this lower than the dfs.namenode.replication.min
is not recommend and/or dangerous for production setups.
When it's not set it takes value from dfs.namenode.replication.min
</description>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
<description>
The default block size for new files, in bytes.
You can use the following suffix (case insensitive):
k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa) to specify the size (such as 128k, 512m, 1g, etc.),
Or provide complete size in bytes (such as 134217728 for 128 MB).
</description>
</property>
<property>
<name>dfs.client.block.write.retries</name>
<value>3</value>
<description>The number of retries for writing blocks to the data nodes,
before we signal failure to the application.
</description>
</property>
<property>
<name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
<value>true</value>
<description>
If there is a datanode/network failure in the write pipeline,
DFSClient will try to remove the failed datanode from the pipeline
and then continue writing with the remaining datanodes. As a result,
the number of datanodes in the pipeline is decreased. The feature is
to add new datanodes to the pipeline.
This is a site-wide property to enable/disable the feature.
When the cluster size is extremely small, e.g. 3 nodes or less, cluster
administrators may want to set the policy to NEVER in the default
configuration file or disable this feature. Otherwise, users may
experience an unusually high rate of pipeline failures since it is
impossible to find new datanodes for replacement.
See also dfs.client.block.write.replace-datanode-on-failure.policy
</description>
</property>
<property>
<name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
<value>DEFAULT</value>
<description>
This property is used only if the value of
dfs.client.block.write.replace-datanode-on-failure.enable is true.
ALWAYS: always add a new datanode when an existing datanode is removed.
NEVER: never add a new datanode.
DEFAULT:
Let r be the replication number.
Let n be the number of existing datanodes.
Add a new datanode only if r is greater than or equal to 3 and either
(1) floor(r/2) is greater than or equal to n; or
(2) r is greater than n and the block is hflushed/appended.
</description>
</property>
<property>
<name>dfs.client.block.write.replace-datanode-on-failure.best-effort</name>
<value>false</value>
<description>
This property is used only if the value of
dfs.client.block.write.replace-datanode-on-failure.enable is true.
Best effort means that the client will try to replace a failed datanode
in write pipeline (provided that the policy is satisfied), however, it
continues the write operation in case that the datanode replacement also
fails.
Suppose the datanode replacement fails.
false: An exception should be thrown so that the write will fail.
true : The write should be resumed with the remaining datandoes.
Note that setting this property to true allows writing to a pipeline
with a smaller number of datanodes. As a result, it increases the
probability of data loss.
</description>
</property>
<property>
<name>dfs.client.block.write.replace-datanode-on-failure.min-replication</name>
<value>0</value>
<description>
The minimum number of replications that are needed to not to fail
the write pipeline if new datanodes can not be found to replace
failed datanodes (could be due to network failure) in the write pipeline.
If the number of the remaining datanodes in the write pipeline is greater
than or equal to this property value, continue writing to the remaining nodes.
Otherwise throw exception.
If this is set to 0, an exception will be thrown, when a replacement
can not be found.
See also dfs.client.block.write.replace-datanode-on-failure.policy
</description>
</property>
<property>
<name>dfs.blockreport.intervalMsec</name>
<value>21600000</value>
<description>Determines block reporting interval in milliseconds.</description>
</property>
<property>
<name>dfs.blockreport.initialDelay</name>
<value>0s</value>
<description>
Delay for first block report in seconds. Support multiple time unit
suffix(case insensitive), as described in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.blockreport.split.threshold</name>
<value>1000000</value>
<description>If the number of blocks on the DataNode is below this
threshold then it will send block reports for all Storage Directories
in a single message.
If the number of blocks exceeds this threshold then the DataNode will
send block reports for each Storage Directory in separate messages.
Set to zero to always split.
</description>
</property>
<property>
<name>dfs.namenode.max.full.block.report.leases</name>
<value>6</value>
<description>The maximum number of leases for full block reports that the
NameNode will issue at any given time. This prevents the NameNode from
being flooded with full block reports that use up all the RPC handler
threads. This number should never be more than the number of RPC handler
threads or less than 1.
</description>
</property>
<property>
<name>dfs.namenode.full.block.report.lease.length.ms</name>
<value>300000</value>
<description>
The number of milliseconds that the NameNode will wait before invalidating
a full block report lease. This prevents a crashed DataNode from
permanently using up a full block report lease.
</description>
</property>
<property>
<name>dfs.datanode.directoryscan.interval</name>
<value>21600s</value>
<description>Interval in seconds for Datanode to scan data directories and
reconcile the difference between blocks in memory and on the disk.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.datanode.directoryscan.threads</name>
<value>1</value>
<description>How many threads should the threadpool used to compile reports
for volumes in parallel have.
</description>
</property>
<property>
<name>dfs.datanode.directoryscan.throttle.limit.ms.per.sec</name>
<value>1000</value>
<description>The report compilation threads are limited to only running for
a given number of milliseconds per second, as configured by the
property. The limit is taken per thread, not in aggregate, e.g. setting
a limit of 100ms for 4 compiler threads will result in each thread being
limited to 100ms, not 25ms.
Note that the throttle does not interrupt the report compiler threads, so the
actual running time of the threads per second will typically be somewhat
higher than the throttle limit, usually by no more than 20%.
Setting this limit to 1000 disables compiler thread throttling. Only
values between 1 and 1000 are valid. Setting an invalid value will result
in the throttle being disabled and an error message being logged. 1000 is
the default setting.
</description>
</property>
<property>
<name>dfs.heartbeat.interval</name>
<value>3s</value>
<description>
Determines datanode heartbeat interval in seconds.
Can use the following suffix (case insensitive):
ms(millis), s(sec), m(min), h(hour), d(day)
to specify the time (such as 2s, 2m, 1h, etc.).
Or provide complete number in seconds (such as 30 for 30 seconds).
</description>
</property>
<property>
<name>dfs.datanode.lifeline.interval.seconds</name>
<value></value>
<description>
Sets the interval in seconds between sending DataNode Lifeline Protocol
messages from the DataNode to the NameNode. The value must be greater than
the value of dfs.heartbeat.interval. If this property is not defined, then
the default behavior is to calculate the interval as 3x the value of
dfs.heartbeat.interval. Note that normal heartbeat processing may cause the
DataNode to postpone sending lifeline messages if they are not required.
Under normal operations with speedy heartbeat processing, it is possible
that no lifeline messages will need to be sent at all. This property has no
effect if dfs.namenode.lifeline.rpc-address is not defined.
</description>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>10</value>
<description>The number of Namenode RPC server threads that listen to
requests from clients.
If dfs.namenode.servicerpc-address is not configured then
Namenode RPC server threads listen to requests from all nodes.
</description>
</property>
<property>
<name>dfs.namenode.service.handler.count</name>
<value>10</value>
<description>The number of Namenode RPC server threads that listen to
requests from DataNodes and from all other non-client nodes.
dfs.namenode.service.handler.count will be valid only if
dfs.namenode.servicerpc-address is configured.
</description>
</property>
<property>
<name>dfs.namenode.lifeline.handler.ratio</name>
<value>0.10</value>
<description>
A ratio applied to the value of dfs.namenode.handler.count, which then
provides the number of RPC server threads the NameNode runs for handling the
lifeline RPC server. For example, if dfs.namenode.handler.count is 100, and
dfs.namenode.lifeline.handler.factor is 0.10, then the NameNode starts
100 * 0.10 = 10 threads for handling the lifeline RPC server. It is common
to tune the value of dfs.namenode.handler.count as a function of the number
of DataNodes in a cluster. Using this property allows for the lifeline RPC
server handler threads to be tuned automatically without needing to touch a
separate property. Lifeline message processing is lightweight, so it is
expected to require many fewer threads than the main NameNode RPC server.
This property is not used if dfs.namenode.lifeline.handler.count is defined,
which sets an absolute thread count. This property has no effect if
dfs.namenode.lifeline.rpc-address is not defined.
</description>
</property>
<property>
<name>dfs.namenode.lifeline.handler.count</name>
<value></value>
<description>
Sets an absolute number of RPC server threads the NameNode runs for handling
the DataNode Lifeline Protocol and HA health check requests from ZKFC. If
this property is defined, then it overrides the behavior of
dfs.namenode.lifeline.handler.ratio. By default, it is not defined. This
property has no effect if dfs.namenode.lifeline.rpc-address is not defined.
</description>
</property>
<property>
<name>dfs.namenode.safemode.threshold-pct</name>
<value>0.999f</value>
<description>
Specifies the percentage of blocks that should satisfy
the minimal replication requirement defined by dfs.namenode.replication.min.
Values less than or equal to 0 mean not to wait for any particular
percentage of blocks before exiting safemode.
Values greater than 1 will make safe mode permanent.
</description>
</property>
<property>
<name>dfs.namenode.safemode.min.datanodes</name>
<value>0</value>
<description>
Specifies the number of datanodes that must be considered alive
before the name node exits safemode.
Values less than or equal to 0 mean not to take the number of live
datanodes into account when deciding whether to remain in safe mode
during startup.
Values greater than the number of datanodes in the cluster
will make safe mode permanent.
</description>
</property>
<property>
<name>dfs.namenode.safemode.extension</name>
<value>30000</value>
<description>
Determines extension of safe mode in milliseconds after the threshold level
is reached. Support multiple time unit suffix (case insensitive), as
described in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.namenode.resource.check.interval</name>
<value>5000</value>
<description>
The interval in milliseconds at which the NameNode resource checker runs.
The checker calculates the number of the NameNode storage volumes whose
available spaces are more than dfs.namenode.resource.du.reserved, and
enters safemode if the number becomes lower than the minimum value
specified by dfs.namenode.resource.checked.volumes.minimum.
</description>
</property>
<property>
<name>dfs.namenode.resource.du.reserved</name>
<value>104857600</value>
<description>
The amount of space to reserve/require for a NameNode storage directory
in bytes. The default is 100MB.
</description>
</property>
<property>
<name>dfs.namenode.resource.checked.volumes</name>
<value></value>
<description>
A list of local directories for the NameNode resource checker to check in
addition to the local edits directories.
</description>
</property>
<property>
<name>dfs.namenode.resource.checked.volumes.minimum</name>
<value>1</value>
<description>
The minimum number of redundant NameNode storage volumes required.
</description>
</property>
<property>
<name>dfs.datanode.balance.bandwidthPerSec</name>
<value>10m</value>
<description>
Specifies the maximum amount of bandwidth that each datanode
can utilize for the balancing purpose in term of
the number of bytes per second. You can use the following
suffix (case insensitive):
k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa)to specify the size
(such as 128k, 512m, 1g, etc.).
Or provide complete size in bytes (such as 134217728 for 128 MB).
</description>
</property>
<property>
<name>dfs.hosts</name>
<value></value>
<description>Names a file that contains a list of hosts that are
permitted to connect to the namenode. The full pathname of the file
must be specified. If the value is empty, all hosts are
permitted.</description>
</property>
<property>
<name>dfs.hosts.exclude</name>
<value></value>
<description>Names a file that contains a list of hosts that are
not permitted to connect to the namenode. The full pathname of the
file must be specified. If the value is empty, no hosts are
excluded.</description>
</property>
<property>
<name>dfs.namenode.max.objects</name>
<value>0</value>
<description>The maximum number of files, directories and blocks
dfs supports. A value of zero indicates no limit to the number
of objects that dfs supports.
</description>
</property>
<property>
<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
<value>true</value>
<description>
If true (the default), then the namenode requires that a connecting
datanode's address must be resolved to a hostname. If necessary, a reverse
DNS lookup is performed. All attempts to register a datanode from an
unresolvable address are rejected.
It is recommended that this setting be left on to prevent accidental
registration of datanodes listed by hostname in the excludes file during a
DNS outage. Only set this to false in environments where there is no
infrastructure to support reverse DNS lookup.
</description>
</property>
<property>
<name>dfs.namenode.decommission.interval</name>
<value>30s</value>
<description>Namenode periodicity in seconds to check if
decommission or maintenance is complete. Support multiple time unit
suffix(case insensitive), as described in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.namenode.decommission.blocks.per.interval</name>
<value>500000</value>
<description>The approximate number of blocks to process per decommission
or maintenance interval, as defined in dfs.namenode.decommission.interval.
</description>
</property>
<property>
<name>dfs.namenode.decommission.max.concurrent.tracked.nodes</name>
<value>100</value>
<description>
The maximum number of decommission-in-progress or
entering-maintenance datanodes nodes that will be tracked at one time by
the namenode. Tracking these datanode consumes additional NN memory
proportional to the number of blocks on the datnode. Having a conservative
limit reduces the potential impact of decommissioning or maintenance of
a large number of nodes at once.
A value of 0 means no limit will be enforced.
</description>
</property>
<property>
<name>dfs.namenode.redundancy.interval.seconds</name>
<value>3s</value>
<description>The periodicity in seconds with which the namenode computes
low redundancy work for datanodes. Support multiple time unit suffix(case insensitive),
as described in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.namenode.accesstime.precision</name>
<value>3600000</value>
<description>The access time for HDFS file is precise upto this value.
The default value is 1 hour. Setting a value of 0 disables
access times for HDFS.
</description>
</property>
<property>
<name>dfs.datanode.plugins</name>
<value></value>
<description>Comma-separated list of datanode plug-ins to be activated.
</description>
</property>
<property>
<name>dfs.namenode.plugins</name>
<value></value>
<description>Comma-separated list of namenode plug-ins to be activated.
</description>
</property>
<property>
<name>dfs.namenode.block-placement-policy.default.prefer-local-node</name>
<value>true</value>
<description>Controls how the default block placement policy places
the first replica of a block. When true, it will prefer the node where
the client is running. When false, it will prefer a node in the same rack
as the client. Setting to false avoids situations where entire copies of
large files end up on a single node, thus creating hotspots.
</description>
</property>
<property>
<name>dfs.stream-buffer-size</name>
<value>4096</value>
<description>The size of buffer to stream files.
The size of this buffer should probably be a multiple of hardware
page size (4096 on Intel x86), and it determines how much data is
buffered during read and write operations.</description>
</property>
<property>
<name>dfs.bytes-per-checksum</name>
<value>512</value>
<description>The number of bytes per checksum. Must not be larger than
dfs.stream-buffer-size</description>
</property>
<property>
<name>dfs.client-write-packet-size</name>
<value>65536</value>
<description>Packet size for clients to write</description>
</property>
<property>
<name>dfs.client.write.exclude.nodes.cache.expiry.interval.millis</name>
<value>600000</value>
<description>The maximum period to keep a DN in the excluded nodes list
at a client. After this period, in milliseconds, the previously excluded node(s) will
be removed automatically from the cache and will be considered good for block allocations
again. Useful to lower or raise in situations where you keep a file open for very long
periods (such as a Write-Ahead-Log (WAL) file) to make the writer tolerant to cluster maintenance
restarts. Defaults to 10 minutes.</description>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/namesecondary</value>
<description>Determines where on the local filesystem the DFS secondary
name node should store the temporary images to merge.
If this is a comma-delimited list of directories then the image is
replicated in all of the directories for redundancy.
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.edits.dir</name>
<value>${dfs.namenode.checkpoint.dir}</value>
<description>Determines where on the local filesystem the DFS secondary
name node should store the temporary edits to merge.
If this is a comma-delimited list of directories then the edits is
replicated in all of the directories for redundancy.
Default value is same as dfs.namenode.checkpoint.dir
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.period</name>
<value>3600s</value>
<description>
The number of seconds between two periodic checkpoints.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.txns</name>
<value>1000000</value>
<description>The Secondary NameNode or CheckpointNode will create a checkpoint
of the namespace every 'dfs.namenode.checkpoint.txns' transactions, regardless
of whether 'dfs.namenode.checkpoint.period' has expired.
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.check.period</name>
<value>60s</value>
<description>The SecondaryNameNode and CheckpointNode will poll the NameNode
every 'dfs.namenode.checkpoint.check.period' seconds to query the number
of uncheckpointed transactions. Support multiple time unit suffix(case insensitive),
as described in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.max-retries</name>
<value>3</value>
<description>The SecondaryNameNode retries failed checkpointing. If the
failure occurs while loading fsimage or replaying edits, the number of
retries is limited by this variable.
</description>
</property>
<property>
<name>dfs.namenode.checkpoint.check.quiet-multiplier</name>
<value>1.5</value>
<description>
Used to calculate the amount of time between retries when in the 'quiet' period
for creating checkpoints (active namenode already has an up-to-date image from another
checkpointer), so we wait a multiplier of the dfs.namenode.checkpoint.check.period before
retrying the checkpoint because another node likely is already managing the checkpoints,
allowing us to save bandwidth to transfer checkpoints that don't need to be used.
</description>
</property>
<property>
<name>dfs.namenode.num.checkpoints.retained</name>
<value>2</value>
<description>The number of image checkpoint files (fsimage_*) that will be retained by
the NameNode and Secondary NameNode in their storage directories. All edit
logs (stored on edits_* files) necessary to recover an up-to-date namespace from the oldest retained
checkpoint will also be retained.
</description>
</property>
<property>
<name>dfs.namenode.num.extra.edits.retained</name>
<value>1000000</value>
<description>The number of extra transactions which should be retained
beyond what is minimally necessary for a NN restart.
It does not translate directly to file's age, or the number of files kept,
but to the number of transactions (here "edits" means transactions).
One edit file may contain several transactions (edits).
During checkpoint, NameNode will identify the total number of edits to retain as extra by
checking the latest checkpoint transaction value, subtracted by the value of this property.
Then, it scans edits files to identify the older ones that don't include the computed range of
retained transactions that are to be kept around, and purges them subsequently.
The retainment can be useful for audit purposes or for an HA setup where a remote Standby Node may have
been offline for some time and need to have a longer backlog of retained
edits in order to start again.
Typically each edit is on the order of a few hundred bytes, so the default
of 1 million edits should be on the order of hundreds of MBs or low GBs.
NOTE: Fewer extra edits may be retained than value specified for this setting
if doing so would mean that more segments would be retained than the number
configured by dfs.namenode.max.extra.edits.segments.retained.
</description>
</property>
<property>
<name>dfs.namenode.max.extra.edits.segments.retained</name>
<value>10000</value>
<description>The maximum number of extra edit log segments which should be retained
beyond what is minimally necessary for a NN restart. When used in conjunction with
dfs.namenode.num.extra.edits.retained, this configuration property serves to cap
the number of extra edits files to a reasonable value.
</description>
</property>
<property>
<name>dfs.namenode.delegation.key.update-interval</name>
<value>86400000</value>
<description>The update interval for master key for delegation tokens
in the namenode in milliseconds.
</description>
</property>
<property>
<name>dfs.namenode.delegation.token.max-lifetime</name>
<value>604800000</value>
<description>The maximum lifetime in milliseconds for which a delegation
token is valid.
</description>
</property>
<property>
<name>dfs.namenode.delegation.token.renew-interval</name>
<value>86400000</value>
<description>The renewal interval for delegation token in milliseconds.
</description>
</property>
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
<description>The number of volumes that are allowed to
fail before a datanode stops offering service. By default
any volume failure will cause a datanode to shutdown.
</description>
</property>
<property>
<name>dfs.image.compress</name>
<value>false</value>
<description>Should the dfs image be compressed
</description>
</property>
<property>
<name>dfs.image.compression.codec</name>
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
<description>If the dfs image is compressed, how should they be compressed
This has to be a codec defined in io.compression.codecs.
</description>
</property>
<property>
<name>dfs.image.transfer.timeout</name>
<value>60000</value>
<description>
Socket timeout for the HttpURLConnection instance used in the image
transfer. This is measured in milliseconds.
This timeout prevents client hangs if the connection is idle
for this configured timeout, during image transfer.
</description>
</property>
<property>
<name>dfs.image.transfer.bandwidthPerSec</name>
<value>0</value>
<description>
Maximum bandwidth used for regular image transfers (instead of
bootstrapping the standby namenode), in bytes per second.
This can help keep normal namenode operations responsive during
checkpointing.
A default value of 0 indicates that throttling is disabled.
The maximum bandwidth used for bootstrapping standby namenode is
configured with dfs.image.transfer-bootstrap-standby.bandwidthPerSec.
</description>
</property>
<property>
<name>dfs.image.transfer-bootstrap-standby.bandwidthPerSec</name>
<value>0</value>
<description>
Maximum bandwidth used for transferring image to bootstrap standby
namenode, in bytes per second.
A default value of 0 indicates that throttling is disabled. This default
value should be used in most cases, to ensure timely HA operations.
The maximum bandwidth used for regular image transfers is configured
with dfs.image.transfer.bandwidthPerSec.
</description>
</property>
<property>
<name>dfs.image.transfer.chunksize</name>
<value>65536</value>
<description>
Chunksize in bytes to upload the checkpoint.
Chunked streaming is used to avoid internal buffering of contents
of image file of huge size.
</description>
</property>
<property>
<name>dfs.edit.log.transfer.timeout</name>
<value>30000</value>
<description>
Socket timeout for edit log transfer in milliseconds. This timeout
should be configured such that normal edit log transfer for journal
node syncing can complete successfully.
</description>
</property>
<property>
<name>dfs.edit.log.transfer.bandwidthPerSec</name>
<value>0</value>
<description>
Maximum bandwidth used for transferring edit log to between journal nodes
for syncing, in bytes per second.
A default value of 0 indicates that throttling is disabled.
</description>
</property>
<property>
<name>dfs.namenode.support.allow.format</name>
<value>true</value>
<description>Does HDFS namenode allow itself to be formatted
You may consider setting this to false for any production
cluster, to avoid any possibility of formatting a running DFS.
</description>
</property>
<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>4096</value>
<description>
Specifies the maximum number of threads to use for transferring data
in and out of the DN.
</description>
</property>
<property>
<name>dfs.datanode.scan.period.hours</name>
<value>504</value>
<description>
If this is positive, the DataNode will not scan any
individual block more than once in the specified scan period.
If this is negative, the block scanner is disabled.
If this is set to zero, then the default value of 504 hours
or 3 weeks is used. Prior versions of HDFS incorrectly documented
that setting this key to zero will disable the block scanner.
</description>
</property>
<property>
<name>dfs.block.scanner.volume.bytes.per.second</name>
<value>1048576</value>
<description>
If this is 0, the DataNode's block scanner will be disabled. If this
is positive, this is the number of bytes per second that the DataNode's
block scanner will try to scan from each volume.
</description>
</property>
<property>
<name>dfs.datanode.readahead.bytes</name>
<value>4194304</value>
<description>
While reading block files, if the Hadoop native libraries are available,
the datanode can use the posix_fadvise system call to explicitly
page data into the operating system buffer cache ahead of the current
reader's position. This can improve performance especially when
disks are highly contended.
This configuration specifies the number of bytes ahead of the current
read position which the datanode will attempt to read ahead. This
feature may be disabled by configuring this property to 0.
If the native libraries are not available, this configuration has no
effect.
</description>
</property>
<property>
<name>dfs.datanode.drop.cache.behind.reads</name>
<value>false</value>
<description>
In some workloads, the data read from HDFS is known to be significantly
large enough that it is unlikely to be useful to cache it in the
operating system buffer cache. In this case, the DataNode may be
configured to automatically purge all data from the buffer cache
after it is delivered to the client. This behavior is automatically
disabled for workloads which read only short sections of a block
(e.g HBase random-IO workloads).
This may improve performance for some workloads by freeing buffer
cache space usage for more cacheable data.
If the Hadoop native libraries are not available, this configuration
has no effect.
</description>
</property>
<property>
<name>dfs.datanode.drop.cache.behind.writes</name>
<value>false</value>
<description>
In some workloads, the data written to HDFS is known to be significantly
large enough that it is unlikely to be useful to cache it in the
operating system buffer cache. In this case, the DataNode may be
configured to automatically purge all data from the buffer cache
after it is written to disk.
This may improve performance for some workloads by freeing buffer
cache space usage for more cacheable data.
If the Hadoop native libraries are not available, this configuration
has no effect.
</description>
</property>
<property>
<name>dfs.datanode.sync.behind.writes</name>
<value>false</value>
<description>
If this configuration is enabled, the datanode will instruct the
operating system to enqueue all written data to the disk immediately
after it is written. This differs from the usual OS policy which
may wait for up to 30 seconds before triggering writeback.
This may improve performance for some workloads by smoothing the
IO profile for data written to disk.
If the Hadoop native libraries are not available, this configuration
has no effect.
</description>
</property>
<property>
<name>dfs.client.failover.max.attempts</name>
<value>15</value>
<description>
Expert only. The number of client failover attempts that should be
made before the failover is considered failed.
</description>
</property>
<property>
<name>dfs.client.failover.sleep.base.millis</name>
<value>500</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the base value used in the failover calculation. The
first failover will retry immediately. The 2nd failover attempt
will delay at least dfs.client.failover.sleep.base.millis
milliseconds. And so on.
</description>
</property>
<property>
<name>dfs.client.failover.sleep.max.millis</name>
<value>15000</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the maximum value to wait between failovers.
Specifically, the time between two failover attempts will not
exceed +/- 50% of dfs.client.failover.sleep.max.millis
milliseconds.
</description>
</property>
<property>
<name>dfs.client.failover.connection.retries</name>
<value>0</value>
<description>
Expert only. Indicates the number of retries a failover IPC client
will make to establish a server connection.
</description>
</property>
<property>
<name>dfs.client.failover.connection.retries.on.timeouts</name>
<value>0</value>
<description>
Expert only. The number of retry attempts a failover IPC client
will make on socket timeout when establishing a server connection.
</description>
</property>
<property>
<name>dfs.client.datanode-restart.timeout</name>
<value>30s</value>
<description>
Expert only. The time to wait, in seconds, from reception of an
datanode shutdown notification for quick restart, until declaring
the datanode dead and invoking the normal recovery mechanisms.
The notification is sent by a datanode when it is being shutdown
using the shutdownDatanode admin command with the upgrade option.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.nameservices</name>
<value></value>
<description>
Comma-separated list of nameservices.
</description>
</property>
<property>
<name>dfs.nameservice.id</name>
<value></value>
<description>
The ID of this nameservice. If the nameservice ID is not
configured or more than one nameservice is configured for
dfs.nameservices it is determined automatically by
matching the local node's address with the configured address.
</description>
</property>
<property>
<name>dfs.internal.nameservices</name>
<value></value>
<description>
Comma-separated list of nameservices that belong to this cluster.
Datanode will report to all the nameservices in this list. By default
this is set to the value of dfs.nameservices.
</description>
</property>
<property>
<name>dfs.ha.namenodes.EXAMPLENAMESERVICE</name>
<value></value>
<description>
The prefix for a given nameservice, contains a comma-separated
list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).
Unique identifiers for each NameNode in the nameservice, delimited by
commas. This will be used by DataNodes to determine all the NameNodes
in the cluster. For example, if you used “mycluster” as the nameservice
ID previously, and you wanted to use “nn1” and “nn2” as the individual
IDs of the NameNodes, you would configure a property
dfs.ha.namenodes.mycluster, and its value "nn1,nn2".
</description>
</property>
<property>
<name>dfs.ha.namenode.id</name>
<value></value>
<description>
The ID of this namenode. If the namenode ID is not configured it
is determined automatically by matching the local node's address
with the configured address.
</description>
</property>
<property>
<name>dfs.ha.log-roll.period</name>
<value>120s</value>
<description>
How often, in seconds, the StandbyNode should ask the active to
roll edit logs. Since the StandbyNode only reads from finalized
log segments, the StandbyNode will only be as up-to-date as how
often the logs are rolled. Note that failover triggers a log roll
so the StandbyNode will be up to date before it becomes active.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.ha.tail-edits.period</name>
<value>60s</value>
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.ha.tail-edits.namenode-retries</name>
<value>3</value>
<description>
Number of retries to use when contacting the namenode when tailing the log.
</description>
</property>
<property>
<name>dfs.ha.tail-edits.rolledits.timeout</name>
<value>60</value>
<description>The timeout in seconds of calling rollEdits RPC on Active NN.
</description>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>false</value>
<description>
Whether automatic failover is enabled. See the HDFS High
Availability documentation for details on automatic HA
configuration.
</description>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
<description>Whether clients should use datanode hostnames when
connecting to datanodes.
</description>
</property>
<property>
<name>dfs.datanode.use.datanode.hostname</name>
<value>false</value>
<description>Whether datanodes should use datanode hostnames when
connecting to other datanodes for data transfer.
</description>
</property>
<property>
<name>dfs.client.local.interfaces</name>
<value></value>
<description>A comma separated list of network interface names to use
for data transfer between the client and datanodes. When creating
a connection to read from or write to a datanode, the client
chooses one of the specified interfaces at random and binds its
socket to the IP of that interface. Individual names may be
specified as either an interface name (eg "eth0"), a subinterface
name (eg "eth0:0"), or an IP address (which may be specified using
CIDR notation to match a range of IPs).
</description>
</property>
<property>
<name>dfs.datanode.shared.file.descriptor.paths</name>
<value>/dev/shm,/tmp</value>
<description>
A comma-separated list of paths to use when creating file descriptors that
will be shared between the DataNode and the DFSClient. Typically we use
/dev/shm, so that the file descriptors will not be written to disk.
Systems that don't have /dev/shm will fall back to /tmp by default.
</description>
</property>
<property>
<name>dfs.short.circuit.shared.memory.watcher.interrupt.check.ms</name>
<value>60000</value>
<description>
The length of time in milliseconds that the short-circuit shared memory
watcher will go between checking for java interruptions sent from other
threads. This is provided mainly for unit tests.
</description>
</property>
<property>
<name>dfs.namenode.kerberos.principal</name>
<value></value>
<description>
The NameNode service principal. This is typically set to
nn/_HOST@REALM.TLD. Each NameNode will substitute _HOST with its
own fully qualified hostname at startup. The _HOST placeholder
allows using the same configuration setting on both NameNodes
in an HA setup.
</description>
</property>
<property>
<name>dfs.namenode.keytab.file</name>
<value></value>
<description>
The keytab file used by each NameNode daemon to login as its
service principal. The principal name is configured with
dfs.namenode.kerberos.principal.
</description>
</property>
<property>
<name>dfs.datanode.kerberos.principal</name>
<value></value>
<description>
The DataNode service principal. This is typically set to
dn/_HOST@REALM.TLD. Each DataNode will substitute _HOST with its
own fully qualified hostname at startup. The _HOST placeholder
allows using the same configuration setting on all DataNodes.
</description>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
<value></value>
<description>
The keytab file used by each DataNode daemon to login as its
service principal. The principal name is configured with
dfs.datanode.kerberos.principal.
</description>
</property>
<property>
<name>dfs.journalnode.kerberos.principal</name>
<value></value>
<description>
The JournalNode service principal. This is typically set to
jn/_HOST@REALM.TLD. Each JournalNode will substitute _HOST with its
own fully qualified hostname at startup. The _HOST placeholder
allows using the same configuration setting on all JournalNodes.
</description>
</property>
<property>
<name>dfs.journalnode.keytab.file</name>
<value></value>
<description>
The keytab file used by each JournalNode daemon to login as its
service principal. The principal name is configured with
dfs.journalnode.kerberos.principal.
</description>
</property>
<property>
<name>dfs.namenode.kerberos.internal.spnego.principal</name>
<value>${dfs.web.authentication.kerberos.principal}</value>
<description>
The server principal used by the NameNode for web UI SPNEGO
authentication when Kerberos security is enabled. This is
typically set to HTTP/_HOST@REALM.TLD The SPNEGO server principal
begins with the prefix HTTP/ by convention.
If the value is '*', the web server will attempt to login with
every principal specified in the keytab file
dfs.web.authentication.kerberos.keytab.
</description>
</property>
<property>
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
<value></value>
<description>
The server principal used by the JournalNode HTTP Server for
SPNEGO authentication when Kerberos security is enabled. This is
typically set to HTTP/_HOST@REALM.TLD. The SPNEGO server principal
begins with the prefix HTTP/ by convention.
If the value is '*', the web server will attempt to login with
every principal specified in the keytab file
dfs.web.authentication.kerberos.keytab.
For most deployments this can be set to ${dfs.web.authentication.kerberos.principal}
i.e use the value of dfs.web.authentication.kerberos.principal.
</description>
</property>
<property>
<name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
<value>${dfs.web.authentication.kerberos.principal}</value>
<description>
The server principal used by the Secondary NameNode for web UI SPNEGO
authentication when Kerberos security is enabled. Like all other
Secondary NameNode settings, it is ignored in an HA setup.
If the value is '*', the web server will attempt to login with
every principal specified in the keytab file
dfs.web.authentication.kerberos.keytab.
</description>
</property>
<property>
<name>dfs.web.authentication.kerberos.principal</name>
<value></value>
<description>
The server principal used by the NameNode for WebHDFS SPNEGO
authentication.
Required when WebHDFS and security are enabled. In most secure clusters this
setting is also used to specify the values for
dfs.namenode.kerberos.internal.spnego.principal and
dfs.journalnode.kerberos.internal.spnego.principal.
</description>
</property>
<property>
<name>dfs.web.authentication.kerberos.keytab</name>
<value></value>
<description>
The keytab file for the principal corresponding to
dfs.web.authentication.kerberos.principal.
</description>
</property>
<property>
<name>dfs.namenode.kerberos.principal.pattern</name>
<value>*</value>
<description>
A client-side RegEx that can be configured to control
allowed realms to authenticate with (useful in cross-realm env.)
</description>
</property>
<property>
<name>dfs.namenode.avoid.read.stale.datanode</name>
<value>false</value>
<description>
Indicate whether or not to avoid reading from "stale" datanodes whose
heartbeat messages have not been received by the namenode
for more than a specified time interval. Stale datanodes will be
moved to the end of the node list returned for reading. See
dfs.namenode.avoid.write.stale.datanode for a similar setting for writes.
</description>
</property>
<property>
<name>dfs.namenode.avoid.write.stale.datanode</name>
<value>false</value>
<description>
Indicate whether or not to avoid writing to "stale" datanodes whose
heartbeat messages have not been received by the namenode
for more than a specified time interval. Writes will avoid using
stale datanodes unless more than a configured ratio
(dfs.namenode.write.stale.datanode.ratio) of datanodes are marked as
stale. See dfs.namenode.avoid.read.stale.datanode for a similar setting
for reads.
</description>
</property>
<property>
<name>dfs.namenode.stale.datanode.interval</name>
<value>30000</value>
<description>
Default time interval in milliseconds for marking a datanode as "stale",
i.e., if the namenode has not received heartbeat msg from a datanode for
more than this time interval, the datanode will be marked and treated
as "stale" by default. The stale interval cannot be too small since
otherwise this may cause too frequent change of stale states.
We thus set a minimum stale interval value (the default value is 3 times
of heartbeat interval) and guarantee that the stale interval cannot be less
than the minimum value. A stale data node is avoided during lease/block
recovery. It can be conditionally avoided for reads (see
dfs.namenode.avoid.read.stale.datanode) and for writes (see
dfs.namenode.avoid.write.stale.datanode).
</description>
</property>
<property>
<name>dfs.namenode.write.stale.datanode.ratio</name>
<value>0.5f</value>
<description>
When the ratio of number stale datanodes to total datanodes marked
is greater than this ratio, stop avoiding writing to stale nodes so
as to prevent causing hotspots.
</description>
</property>
<property>
<name>dfs.namenode.invalidate.work.pct.per.iteration</name>
<value>0.32f</value>
<description>
*Note*: Advanced property. Change with caution.
This determines the percentage amount of block
invalidations (deletes) to do over a single DN heartbeat
deletion command. The final deletion count is determined by applying this
percentage to the number of live nodes in the system.
The resultant number is the number of blocks from the deletion list
chosen for proper invalidation over a single heartbeat of a single DN.
Value should be a positive, non-zero percentage in float notation (X.Yf),
with 1.0f meaning 100%.
</description>
</property>
<property>
<name>dfs.namenode.replication.work.multiplier.per.iteration</name>
<value>2</value>
<description>
*Note*: Advanced property. Change with caution.
This determines the total amount of block transfers to begin in
parallel at a DN, for replication, when such a command list is being
sent over a DN heartbeat by the NN. The actual number is obtained by
multiplying this multiplier with the total number of live nodes in the
cluster. The result number is the number of blocks to begin transfers
immediately for, per DN heartbeat. This number can be any positive,
non-zero integer.
</description>
</property>
<property>
<name>nfs.server.port</name>
<value>2049</value>
<description>
Specify the port number used by Hadoop NFS.
</description>
</property>
<property>
<name>nfs.mountd.port</name>
<value>4242</value>
<description>
Specify the port number used by Hadoop mount daemon.
</description>
</property>
<property>
<name>nfs.dump.dir</name>
<value>/tmp/.hdfs-nfs</value>
<description>
This directory is used to temporarily save out-of-order writes before
writing to HDFS. For each file, the out-of-order writes are dumped after
they are accumulated to exceed certain threshold (e.g., 1MB) in memory.
One needs to make sure the directory has enough space.
</description>
</property>
<property>
<name>nfs.rtmax</name>
<value>1048576</value>
<description>This is the maximum size in bytes of a READ request
supported by the NFS gateway. If you change this, make sure you
also update the nfs mount's rsize(add rsize= # of bytes to the
mount directive).
</description>
</property>
<property>
<name>nfs.wtmax</name>
<value>1048576</value>
<description>This is the maximum size in bytes of a WRITE request
supported by the NFS gateway. If you change this, make sure you
also update the nfs mount's wsize(add wsize= # of bytes to the
mount directive).
</description>
</property>
<property>
<name>nfs.keytab.file</name>
<value></value>
<description>
*Note*: Advanced property. Change with caution.
This is the path to the keytab file for the hdfs-nfs gateway.
This is required when the cluster is kerberized.
</description>
</property>
<property>
<name>nfs.kerberos.principal</name>
<value></value>
<description>
*Note*: Advanced property. Change with caution.
This is the name of the kerberos principal. This is required when
the cluster is kerberized.It must be of this format:
nfs-gateway-user/nfs-gateway-host@kerberos-realm
</description>
</property>
<property>
<name>nfs.allow.insecure.ports</name>
<value>true</value>
<description>
When set to false, client connections originating from unprivileged ports
(those above 1023) will be rejected. This is to ensure that clients
connecting to this NFS Gateway must have had root privilege on the machine
where they're connecting from.
</description>
</property>
<property>
<name>hadoop.fuse.connection.timeout</name>
<value>300</value>
<description>
The minimum number of seconds that we'll cache libhdfs connection objects
in fuse_dfs. Lower values will result in lower memory consumption; higher
values may speed up access by avoiding the overhead of creating new
connection objects.
</description>
</property>
<property>
<name>hadoop.fuse.timer.period</name>
<value>5</value>
<description>
The number of seconds between cache expiry checks in fuse_dfs. Lower values
will result in fuse_dfs noticing changes to Kerberos ticket caches more
quickly.
</description>
</property>
<property>
<name>dfs.namenode.metrics.logger.period.seconds</name>
<value>600</value>
<description>
This setting controls how frequently the NameNode logs its metrics. The
logging configuration must also define one or more appenders for
NameNodeMetricsLog for the metrics to be logged.
NameNode metrics logging is disabled if this value is set to zero or
less than zero.
</description>
</property>
<property>
<name>dfs.datanode.metrics.logger.period.seconds</name>
<value>600</value>
<description>
This setting controls how frequently the DataNode logs its metrics. The
logging configuration must also define one or more appenders for
DataNodeMetricsLog for the metrics to be logged.
DataNode metrics logging is disabled if this value is set to zero or
less than zero.
</description>
</property>
<property>
<name>dfs.metrics.percentiles.intervals</name>
<value></value>
<description>
Comma-delimited set of integers denoting the desired rollover intervals
(in seconds) for percentile latency metrics on the Namenode and Datanode.
By default, percentile latency metrics are disabled.
</description>
</property>
<property>
<name>dfs.datanode.peer.stats.enabled</name>
<value>false</value>
<description>
A switch to turn on/off tracking DataNode peer statistics.
</description>
</property>
<property>
<name>dfs.datanode.outliers.report.interval</name>
<value>30m</value>
<description>
This setting controls how frequently DataNodes will report their peer
latencies to the NameNode via heartbeats. This setting supports
multiple time unit suffixes as described in dfs.heartbeat.interval.
If no suffix is specified then milliseconds is assumed.
It is ignored if dfs.datanode.peer.stats.enabled is false.
</description>
</property>
<property>
<name>dfs.datanode.fileio.profiling.sampling.percentage</name>
<value>0</value>
<description>
This setting controls the percentage of file I/O events which will be
profiled for DataNode disk statistics. The default value of 0 disables
disk statistics. Set to an integer value between 1 and 100 to enable disk
statistics.
</description>
</property>
<property>
<name>hadoop.user.group.metrics.percentiles.intervals</name>
<value></value>
<description>
A comma-separated list of the granularity in seconds for the metrics
which describe the 50/75/90/95/99th percentile latency for group resolution
in milliseconds.
By default, percentile latency metrics are disabled.
</description>
</property>
<property>
<name>dfs.encrypt.data.transfer</name>
<value>false</value>
<description>
Whether or not actual block data that is read/written from/to HDFS should
be encrypted on the wire. This only needs to be set on the NN and DNs,
clients will deduce this automatically. It is possible to override this setting
per connection by specifying custom logic via dfs.trustedchannel.resolver.class.
</description>
</property>
<property>
<name>dfs.encrypt.data.transfer.algorithm</name>
<value></value>
<description>
This value may be set to either "3des" or "rc4". If nothing is set, then
the configured JCE default on the system is used (usually 3DES.) It is
widely believed that 3DES is more cryptographically secure, but RC4 is
substantially faster.
Note that if AES is supported by both the client and server then this
encryption algorithm will only be used to initially transfer keys for AES.
(See dfs.encrypt.data.transfer.cipher.suites.)
</description>
</property>
<property>
<name>dfs.encrypt.data.transfer.cipher.suites</name>
<value></value>
<description>
This value may be either undefined or AES/CTR/NoPadding. If defined, then
dfs.encrypt.data.transfer uses the specified cipher suite for data
encryption. If not defined, then only the algorithm specified in
dfs.encrypt.data.transfer.algorithm is used. By default, the property is
not defined.
</description>
</property>
<property>
<name>dfs.encrypt.data.transfer.cipher.key.bitlength</name>
<value>128</value>
<description>
The key bitlength negotiated by dfsclient and datanode for encryption.
This value may be set to either 128, 192 or 256.
</description>
</property>
<property>
<name>dfs.trustedchannel.resolver.class</name>
<value></value>
<description>
TrustedChannelResolver is used to determine whether a channel
is trusted for plain data transfer. The TrustedChannelResolver is
invoked on both client and server side. If the resolver indicates
that the channel is trusted, then the data transfer will not be
encrypted even if dfs.encrypt.data.transfer is set to true. The
default implementation returns false indicating that the channel
is not trusted.
</description>
</property>
<property>
<name>dfs.data.transfer.protection</name>
<value></value>
<description>
A comma-separated list of SASL protection values used for secured
connections to the DataNode when reading or writing block data. Possible
values are authentication, integrity and privacy. authentication means
authentication only and no integrity or privacy; integrity implies
authentication and integrity are enabled; and privacy implies all of
authentication, integrity and privacy are enabled. If
dfs.encrypt.data.transfer is set to true, then it supersedes the setting for
dfs.data.transfer.protection and enforces that all connections must use a
specialized encrypted SASL handshake. This property is ignored for
connections to a DataNode listening on a privileged port. In this case, it
is assumed that the use of a privileged port establishes sufficient trust.
</description>
</property>
<property>
<name>dfs.data.transfer.saslproperties.resolver.class</name>
<value></value>
<description>
SaslPropertiesResolver used to resolve the QOP used for a connection to the
DataNode when reading or writing block data. If not specified, the value of
hadoop.security.saslproperties.resolver.class is used as the default value.
</description>
</property>
<property>
<name>dfs.journalnode.rpc-address</name>
<value>0.0.0.0:8485</value>
<description>
The JournalNode RPC server address and port.
</description>
</property>
<property>
<name>dfs.journalnode.rpc-bind-host</name>
<value></value>
<description>
The actual address the RPC server will bind to. If this optional address is
set, it overrides only the hostname portion of dfs.journalnode.rpc-address.
This is useful for making the JournalNode listen on all interfaces by
setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.journalnode.http-address</name>
<value>0.0.0.0:8480</value>
<description>
The address and port the JournalNode HTTP server listens on.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.journalnode.http-bind-host</name>
<value></value>
<description>
The actual address the HTTP server will bind to. If this optional address
is set, it overrides only the hostname portion of
dfs.journalnode.http-address. This is useful for making the JournalNode
HTTP server listen on allinterfaces by setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.journalnode.https-address</name>
<value>0.0.0.0:8481</value>
<description>
The address and port the JournalNode HTTPS server listens on.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>dfs.journalnode.https-bind-host</name>
<value></value>
<description>
The actual address the HTTP server will bind to. If this optional address
is set, it overrides only the hostname portion of
dfs.journalnode.https-address. This is useful for making the JournalNode
HTTP server listen on all interfaces by setting it to 0.0.0.0.
</description>
</property>
<property>
<name>dfs.namenode.audit.loggers</name>
<value>default</value>
<description>
List of classes implementing audit loggers that will receive audit events.
These should be implementations of org.apache.hadoop.hdfs.server.namenode.AuditLogger.
The special value "default" can be used to reference the default audit
logger, which uses the configured log system. Installing custom audit loggers
may affect the performance and stability of the NameNode. Refer to the custom
logger's documentation for more details.
</description>
</property>
<property>
<name>dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold</name>
<value>10737418240</value> <!-- 10 GB -->
<description>
Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to
org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy.
This setting controls how much DN volumes are allowed to differ in terms of
bytes of free disk space before they are considered imbalanced. If the free
space of all the volumes are within this range of each other, the volumes
will be considered balanced and block assignments will be done on a pure
round robin basis.
</description>
</property>
<property>
<name>dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction</name>
<value>0.75f</value>
<description>
Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to
org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy.
This setting controls what percentage of new block allocations will be sent
to volumes with more available disk space than others. This setting should
be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should
be no reason to prefer that volumes with less available disk space receive
more block allocations.
</description>
</property>
<property>
<name>dfs.namenode.edits.noeditlogchannelflush</name>
<value>false</value>
<description>
Specifies whether to flush edit log file channel. When set, expensive
FileChannel#force calls are skipped and synchronous disk writes are
enabled instead by opening the edit log file with RandomAccessFile("rws")
flags. This can significantly improve the performance of edit log writes
on the Windows platform.
Note that the behavior of the "rws" flags is platform and hardware specific
and might not provide the same level of guarantees as FileChannel#force.
For example, the write will skip the disk-cache on SAS and SCSI devices
while it might not on SATA devices. This is an expert level setting,
change with caution.
</description>
</property>
<property>
<name>dfs.client.cache.drop.behind.writes</name>
<value></value>
<description>
Just like dfs.datanode.drop.cache.behind.writes, this setting causes the
page cache to be dropped behind HDFS writes, potentially freeing up more
memory for other uses. Unlike dfs.datanode.drop.cache.behind.writes, this
is a client-side setting rather than a setting for the entire datanode.
If present, this setting will override the DataNode default.
If the native libraries are not available to the DataNode, this
configuration has no effect.
</description>
</property>
<property>
<name>dfs.client.cache.drop.behind.reads</name>
<value></value>
<description>
Just like dfs.datanode.drop.cache.behind.reads, this setting causes the
page cache to be dropped behind HDFS reads, potentially freeing up more
memory for other uses. Unlike dfs.datanode.drop.cache.behind.reads, this
is a client-side setting rather than a setting for the entire datanode. If
present, this setting will override the DataNode default.
If the native libraries are not available to the DataNode, this
configuration has no effect.
</description>
</property>
<property>
<name>dfs.client.cache.readahead</name>
<value></value>
<description>
When using remote reads, this setting causes the datanode to
read ahead in the block file using posix_fadvise, potentially decreasing
I/O wait times. Unlike dfs.datanode.readahead.bytes, this is a client-side
setting rather than a setting for the entire datanode. If present, this
setting will override the DataNode default.
When using local reads, this setting determines how much readahead we do in
BlockReaderLocal.
If the native libraries are not available to the DataNode, this
configuration has no effect.
</description>
</property>
<property>
<name>dfs.client.server-defaults.validity.period.ms</name>
<value>3600000</value>
<description>
The amount of milliseconds after which cached server defaults are updated.
By default this parameter is set to 1 hour.
</description>
</property>
<property>
<name>dfs.namenode.enable.retrycache</name>
<value>true</value>
<description>
This enables the retry cache on the namenode. Namenode tracks for
non-idempotent requests the corresponding response. If a client retries the
request, the response from the retry cache is sent. Such operations
are tagged with annotation @AtMostOnce in namenode protocols. It is
recommended that this flag be set to true. Setting it to false, will result
in clients getting failure responses to retried request. This flag must
be enabled in HA setup for transparent fail-overs.
The entries in the cache have expiration time configurable
using dfs.namenode.retrycache.expirytime.millis.
</description>
</property>
<property>
<name>dfs.namenode.retrycache.expirytime.millis</name>
<value>600000</value>
<description>
The time for which retry cache entries are retained.
</description>
</property>
<property>
<name>dfs.namenode.retrycache.heap.percent</name>
<value>0.03f</value>
<description>
This parameter configures the heap size allocated for retry cache
(excluding the response cached). This corresponds to approximately
4096 entries for every 64MB of namenode process java heap size.
Assuming retry cache entry expiration time (configured using
dfs.namenode.retrycache.expirytime.millis) of 10 minutes, this
enables retry cache to support 7 operations per second sustained
for 10 minutes. As the heap size is increased, the operation rate
linearly increases.
</description>
</property>
<property>
<name>dfs.client.mmap.enabled</name>
<value>true</value>
<description>
If this is set to false, the client won't attempt to perform memory-mapped reads.
</description>
</property>
<property>
<name>dfs.client.mmap.cache.size</name>
<value>256</value>
<description>
When zero-copy reads are used, the DFSClient keeps a cache of recently used
memory mapped regions. This parameter controls the maximum number of
entries that we will keep in that cache.
The larger this number is, the more file descriptors we will potentially
use for memory-mapped files. mmaped files also use virtual address space.
You may need to increase your ulimit virtual address space limits before
increasing the client mmap cache size.
Note that you can still do zero-copy reads when this size is set to 0.
</description>
</property>
<property>
<name>dfs.client.mmap.cache.timeout.ms</name>
<value>3600000</value>
<description>
The minimum length of time that we will keep an mmap entry in the cache
between uses. If an entry is in the cache longer than this, and nobody
uses it, it will be removed by a background thread.
</description>
</property>
<property>
<name>dfs.client.mmap.retry.timeout.ms</name>
<value>300000</value>
<description>
The minimum amount of time that we will wait before retrying a failed mmap
operation.
</description>
</property>
<property>
<name>dfs.client.short.circuit.replica.stale.threshold.ms</name>
<value>1800000</value>
<description>
The maximum amount of time that we will consider a short-circuit replica to
be valid, if there is no communication from the DataNode. After this time
has elapsed, we will re-fetch the short-circuit replica even if it is in
the cache.
</description>
</property>
<property>
<name>dfs.namenode.path.based.cache.block.map.allocation.percent</name>
<value>0.25</value>
<description>
The percentage of the Java heap which we will allocate to the cached blocks
map. The cached blocks map is a hash map which uses chained hashing.
Smaller maps may be accessed more slowly if the number of cached blocks is
large; larger maps will consume more memory.
</description>
</property>
<property>
<name>dfs.datanode.max.locked.memory</name>
<value>0</value>
<description>
The amount of memory in bytes to use for caching of block replicas in
memory on the datanode. The datanode's maximum locked memory soft ulimit
(RLIMIT_MEMLOCK) must be set to at least this value, else the datanode
will abort on startup.
By default, this parameter is set to 0, which disables in-memory caching.
If the native libraries are not available to the DataNode, this
configuration has no effect.
</description>
</property>
<property>
<name>dfs.namenode.list.cache.directives.num.responses</name>
<value>100</value>
<description>
This value controls the number of cache directives that the NameNode will
send over the wire in response to a listDirectives RPC.
</description>
</property>
<property>
<name>dfs.namenode.list.cache.pools.num.responses</name>
<value>100</value>
<description>
This value controls the number of cache pools that the NameNode will
send over the wire in response to a listPools RPC.
</description>
</property>
<property>
<name>dfs.namenode.path.based.cache.refresh.interval.ms</name>
<value>30000</value>
<description>
The amount of milliseconds between subsequent path cache rescans. Path
cache rescans are when we calculate which blocks should be cached, and on
what datanodes.
By default, this parameter is set to 30 seconds.
</description>
</property>
<property>
<name>dfs.namenode.path.based.cache.retry.interval.ms</name>
<value>30000</value>
<description>
When the NameNode needs to uncache something that is cached, or cache
something that is not cached, it must direct the DataNodes to do so by
sending a DNA_CACHE or DNA_UNCACHE command in response to a DataNode
heartbeat. This parameter controls how frequently the NameNode will
resend these commands.
</description>
</property>
<property>
<name>dfs.datanode.fsdatasetcache.max.threads.per.volume</name>
<value>4</value>
<description>
The maximum number of threads per volume to use for caching new data
on the datanode. These threads consume both I/O and CPU. This can affect
normal datanode operations.
</description>
</property>
<property>
<name>dfs.cachereport.intervalMsec</name>
<value>10000</value>
<description>
Determines cache reporting interval in milliseconds. After this amount of
time, the DataNode sends a full report of its cache state to the NameNode.
The NameNode uses the cache report to update its map of cached blocks to
DataNode locations.
This configuration has no effect if in-memory caching has been disabled by
setting dfs.datanode.max.locked.memory to 0 (which is the default).
If the native libraries are not available to the DataNode, this
configuration has no effect.
</description>
</property>
<property>
<name>dfs.namenode.edit.log.autoroll.multiplier.threshold</name>
<value>2.0</value>
<description>
Determines when an active namenode will roll its own edit log.
The actual threshold (in number of edits) is determined by multiplying
this value by dfs.namenode.checkpoint.txns.
This prevents extremely large edit files from accumulating on the active
namenode, which can cause timeouts during namenode startup and pose an
administrative hassle. This behavior is intended as a failsafe for when
the standby or secondary namenode fail to roll the edit log by the normal
checkpoint threshold.
</description>
</property>
<property>
<name>dfs.namenode.edit.log.autoroll.check.interval.ms</name>
<value>300000</value>
<description>
How often an active namenode will check if it needs to roll its edit log,
in milliseconds.
</description>
</property>
<property>
<name>dfs.webhdfs.user.provider.user.pattern</name>
<value>^[A-Za-z_][A-Za-z0-9._-]*[$]$</value>
<description>
Valid pattern for user and group names for webhdfs, it must be a valid java regex.
</description>
</property>
<property>
<name>dfs.webhdfs.acl.provider.permission.pattern</name>
<value>^(default:)(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})(,(default:)(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3}))*$</value>
<description>
Valid pattern for user and group names in webhdfs acl operations, it must be a valid java regex.
</description>
</property>
<property>
<name>dfs.webhdfs.socket.connect-timeout</name>
<value>60s</value>
<description>
Socket timeout for connecting to WebHDFS servers. This prevents a
WebHDFS client from hanging if the server hostname is
misconfigured, or the server does not response before the timeout
expires. Value is followed by a unit specifier: ns, us, ms, s, m,
h, d for nanoseconds, microseconds, milliseconds, seconds,
minutes, hours, days respectively. Values should provide units,
but milliseconds are assumed.
</description>
</property>
<property>
<name>dfs.webhdfs.socket.read-timeout</name>
<value>60s</value>
<description>
Socket timeout for reading data from WebHDFS servers. This
prevents a WebHDFS client from hanging if the server stops sending
data. Value is followed by a unit specifier: ns, us, ms, s, m, h,
d for nanoseconds, microseconds, milliseconds, seconds, minutes,
hours, days respectively. Values should provide units,
but milliseconds are assumed.
</description>
</property>
<property>
<name>dfs.client.context</name>
<value>default</value>
<description>
The name of the DFSClient context that we should use. Clients that share
a context share a socket cache and short-circuit cache, among other things.
You should only change this if you don't want to share with another set of
threads.
</description>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>false</value>
<description>
This configuration parameter turns on short-circuit local reads.
</description>
</property>
<property>
<name>dfs.client.socket.send.buffer.size</name>
<value>0</value>
<description>
Socket send buffer size for a write pipeline in DFSClient side.
This may affect TCP connection throughput.
If it is set to zero or negative value,
no buffer size will be set explicitly,
thus enable tcp auto-tuning on some system.
The default value is 0.
</description>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value></value>
<description>
Optional. This is a path to a UNIX domain socket that will be used for
communication between the DataNode and local HDFS clients.
If the string "_PORT" is present in this path, it will be replaced by the
TCP port of the DataNode.
</description>
</property>
<property>
<name>dfs.domain.socket.disable.interval.seconds</name>
<value>600</value>
<description>
The interval that a DataNode is disabled for future Short-Circuit Reads,
after an error happens during a Short-Circuit Read. Setting this to 0 will
not disable Short-Circuit Reads at all after errors happen. Negative values
are invalid.
</description>
</property>
<property>
<name>dfs.client.read.shortcircuit.skip.checksum</name>
<value>false</value>
<description>
If this configuration parameter is set,
short-circuit local reads will skip checksums.
This is normally not recommended,
but it may be useful for special setups.
You might consider using this
if you are doing your own checksumming outside of HDFS.
</description>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
<value>256</value>
<description>
The DFSClient maintains a cache of recently opened file descriptors.
This parameter controls the maximum number of file descriptors in the cache.
Setting this higher will use more file descriptors,
but potentially provide better performance on workloads
involving lots of seeks.
</description>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.expiry.ms</name>
<value>300000</value>
<description>
This controls the minimum amount of time
file descriptors need to sit in the client cache context
before they can be closed for being inactive for too long.
</description>
</property>
<property>
<name>dfs.datanode.shared.file.descriptor.paths</name>
<value>/dev/shm,/tmp</value>
<description>
Comma separated paths to the directory on which
shared memory segments are created.
The client and the DataNode exchange information via
this shared memory segment.
It tries paths in order until creation of shared memory segment succeeds.
</description>
</property>
<property>
<name>dfs.namenode.audit.log.debug.cmdlist</name>
<value></value>
<description>
A comma separated list of NameNode commands that are written to the HDFS
namenode audit log only if the audit log level is debug.
</description>
</property>
<property>
<name>dfs.client.use.legacy.blockreader.local</name>
<value>false</value>
<description>
Legacy short-circuit reader implementation based on HDFS-2246 is used
if this configuration parameter is true.
This is for the platforms other than Linux
where the new implementation based on HDFS-347 is not available.
</description>
</property>
<property>
<name>dfs.block.local-path-access.user</name>
<value></value>
<description>
Comma separated list of the users allowed to open block files
on legacy short-circuit local read.
</description>
</property>
<property>
<name>dfs.client.domain.socket.data.traffic</name>
<value>false</value>
<description>
This control whether we will try to pass normal data traffic
over UNIX domain socket rather than over TCP socket
on node-local data transfer.
This is currently experimental and turned off by default.
</description>
</property>
<property>
<name>dfs.namenode.reject-unresolved-dn-topology-mapping</name>
<value>false</value>
<description>
If the value is set to true, then namenode will reject datanode
registration if the topology mapping for a datanode is not resolved and
NULL is returned (script defined by net.topology.script.file.name fails
to execute). Otherwise, datanode will be registered and the default rack
will be assigned as the topology path. Topology paths are important for
data resiliency, since they define fault domains. Thus it may be unwanted
behavior to allow datanode registration with the default rack if the
resolving topology failed.
</description>
</property>
<property>
<name>dfs.namenode.xattrs.enabled</name>
<value>true</value>
<description>
Whether support for extended attributes is enabled on the NameNode.
</description>
</property>
<property>
<name>dfs.namenode.fs-limits.max-xattrs-per-inode</name>
<value>32</value>
<description>
Maximum number of extended attributes per inode.
</description>
</property>
<property>
<name>dfs.namenode.fs-limits.max-xattr-size</name>
<value>16384</value>
<description>
The maximum combined size of the name and value of an extended attribute
in bytes. It should be larger than 0, and less than or equal to maximum
size hard limit which is 32768.
</description>
</property>
<property>
<name>dfs.client.slow.io.warning.threshold.ms</name>
<value>30000</value>
<description>The threshold in milliseconds at which we will log a slow
io warning in a dfsclient. By default, this parameter is set to 30000
milliseconds (30 seconds).
</description>
</property>
<property>
<name>dfs.datanode.slow.io.warning.threshold.ms</name>
<value>300</value>
<description>The threshold in milliseconds at which we will log a slow
io warning in a datanode. By default, this parameter is set to 300
milliseconds.
</description>
</property>
<property>
<name>dfs.namenode.lease-recheck-interval-ms</name>
<value>2000</value>
<description>During the release of lease a lock is hold that make any
operations on the namenode stuck. In order to not block them during
a too long duration we stop releasing lease after this max lock limit.
</description>
</property>
<property>
<name>dfs.namenode.max-lock-hold-to-release-lease-ms</name>
<value>25</value>
<description>During the release of lease a lock is hold that make any
operations on the namenode stuck. In order to not block them during
a too long duration we stop releasing lease after this max lock limit.
</description>
</property>
<property>
<name>dfs.namenode.write-lock-reporting-threshold-ms</name>
<value>5000</value>
<description>When a write lock is held on the namenode for a long time,
this will be logged as the lock is released. This sets how long the
lock must be held for logging to occur.
</description>
</property>
<property>
<name>dfs.namenode.read-lock-reporting-threshold-ms</name>
<value>5000</value>
<description>When a read lock is held on the namenode for a long time,
this will be logged as the lock is released. This sets how long the
lock must be held for logging to occur.
</description>
</property>
<property>
<name>dfs.namenode.lock.detailed-metrics.enabled</name>
<value>false</value>
<description>If true, the namenode will keep track of how long various
operations hold the Namesystem lock for and emit this as metrics. These
metrics have names of the form FSN(Read|Write)LockNanosOperationName,
where OperationName denotes the name of the operation that initiated the
lock hold (this will be OTHER for certain uncategorized operations) and
they export the hold time values in nanoseconds.
</description>
</property>
<property>
<name>dfs.namenode.fslock.fair</name>
<value>true</value>
<description>If this is true, the FS Namesystem lock will be used in Fair mode,
which will help to prevent writer threads from being starved, but can provide
lower lock throughput. See java.util.concurrent.locks.ReentrantReadWriteLock
for more information on fair/non-fair locks.
</description>
</property>
<property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>0</value>
<description>The delay in seconds at which we will pause the blocks deletion
after Namenode startup. By default it's disabled.
In the case a directory has large number of directories and files are
deleted, suggested delay is one hour to give the administrator enough time
to notice large number of pending deletion blocks and take corrective
action.
</description>
</property>
<property>
<name>dfs.datanode.block.id.layout.upgrade.threads</name>
<value>12</value>
<description>The number of threads to use when creating hard links from
current to previous blocks during upgrade of a DataNode to block ID-based
block layout (see HDFS-6482 for details on the layout).</description>
</property>
<property>
<name>dfs.namenode.list.encryption.zones.num.responses</name>
<value>100</value>
<description>When listing encryption zones, the maximum number of zones
that will be returned in a batch. Fetching the list incrementally in
batches improves namenode performance.
</description>
</property>
<property>
<name>dfs.namenode.list.reencryption.status.num.responses</name>
<value>100</value>
<description>When listing re-encryption status, the maximum number of zones
that will be returned in a batch. Fetching the list incrementally in
batches improves namenode performance.
</description>
</property>
<property>
<name>dfs.namenode.list.openfiles.num.responses</name>
<value>1000</value>
<description>
When listing open files, the maximum number of open files that will be
returned in a single batch. Fetching the list incrementally in batches
improves namenode performance.
</description>
</property>
<property>
<name>dfs.namenode.edekcacheloader.interval.ms</name>
<value>1000</value>
<description>When KeyProvider is configured, the interval time of warming
up edek cache on NN starts up / becomes active. All edeks will be loaded
from KMS into provider cache. The edek cache loader will try to warm up the
cache until succeed or NN leaves active state.
</description>
</property>
<property>
<name>dfs.namenode.edekcacheloader.initial.delay.ms</name>
<value>3000</value>
<description>When KeyProvider is configured, the time delayed until the first
attempt to warm up edek cache on NN start up / become active.
</description>
</property>
<property>
<name>dfs.namenode.reencrypt.sleep.interval</name>
<value>1m</value>
<description>Interval the re-encrypt EDEK thread sleeps in the main loop. The
interval accepts units. If none given, millisecond is assumed.
</description>
</property>
<property>
<name>dfs.namenode.reencrypt.batch.size</name>
<value>1000</value>
<description>How many EDEKs should the re-encrypt thread process in one batch.
</description>
</property>
<property>
<name>dfs.namenode.reencrypt.throttle.limit.handler.ratio</name>
<value>1.0</value>
<description>Throttling ratio for the re-encryption, indicating what fraction
of time should the re-encrypt handler thread work under NN read lock.
Larger than 1.0 values are interpreted as 1.0. Negative value or 0 are
invalid values and will fail NN startup.
</description>
</property>
<property>
<name>dfs.namenode.reencrypt.throttle.limit.updater.ratio</name>
<value>1.0</value>
<description>Throttling ratio for the re-encryption, indicating what fraction
of time should the re-encrypt updater thread work under NN write lock.
Larger than 1.0 values are interpreted as 1.0. Negative value or 0 are
invalid values and will fail NN startup.
</description>
</property>
<property>
<name>dfs.namenode.reencrypt.edek.threads</name>
<value>10</value>
<description>Maximum number of re-encrypt threads to contact the KMS
and re-encrypt the edeks.
</description>
</property>
<property>
<name>dfs.namenode.inotify.max.events.per.rpc</name>
<value>1000</value>
<description>Maximum number of events that will be sent to an inotify client
in a single RPC response. The default value attempts to amortize away
the overhead for this RPC while avoiding huge memory requirements for the
client and NameNode (1000 events should consume no more than 1 MB.)
</description>
</property>
<property>
<name>dfs.user.home.dir.prefix</name>
<value>/user</value>
<description>The directory to prepend to user name to get the user's
home direcotry.
</description>
</property>
<property>
<name>dfs.datanode.cache.revocation.timeout.ms</name>
<value>900000</value>
<description>When the DFSClient reads from a block file which the DataNode is
caching, the DFSClient can skip verifying checksums. The DataNode will
keep the block file in cache until the client is done. If the client takes
an unusually long time, though, the DataNode may need to evict the block
file from the cache anyway. This value controls how long the DataNode will
wait for the client to release a replica that it is reading without
checksums.
</description>
</property>
<property>
<name>dfs.datanode.cache.revocation.polling.ms</name>
<value>500</value>
<description>How often the DataNode should poll to see if the clients have
stopped using a replica that the DataNode wants to uncache.
</description>
</property>
<property>
<name>dfs.storage.policy.enabled</name>
<value>true</value>
<description>
Allow users to change the storage policy on files and directories.
</description>
</property>
<property>
<name>dfs.namenode.legacy-oiv-image.dir</name>
<value></value>
<description>Determines where to save the namespace in the old fsimage format
during checkpointing by standby NameNode or SecondaryNameNode. Users can
dump the contents of the old format fsimage by oiv_legacy command. If
the value is not specified, old format fsimage will not be saved in
checkpoint.
</description>
</property>
<property>
<name>dfs.namenode.top.enabled</name>
<value>true</value>
<description>Enable nntop: reporting top users on namenode
</description>
</property>
<property>
<name>dfs.namenode.top.window.num.buckets</name>
<value>10</value>
<description>Number of buckets in the rolling window implementation of nntop
</description>
</property>
<property>
<name>dfs.namenode.top.num.users</name>
<value>10</value>
<description>Number of top users returned by the top tool
</description>
</property>
<property>
<name>dfs.namenode.top.windows.minutes</name>
<value>1,5,25</value>
<description>comma separated list of nntop reporting periods in minutes
</description>
</property>
<property>
<name>dfs.webhdfs.ugi.expire.after.access</name>
<value>600000</value>
<description>How long in milliseconds after the last access
the cached UGI will expire. With 0, never expire.
</description>
</property>
<property>
<name>dfs.namenode.blocks.per.postponedblocks.rescan</name>
<value>10000</value>
<description>Number of blocks to rescan for each iteration of
postponedMisreplicatedBlocks.
</description>
</property>
<property>
<name>dfs.datanode.block-pinning.enabled</name>
<value>false</value>
<description>Whether pin blocks on favored DataNode.</description>
</property>
<property>
<name>dfs.client.block.write.locateFollowingBlock.initial.delay.ms</name>
<value>400</value>
<description>The initial delay (unit is ms) for locateFollowingBlock,
the delay time will increase exponentially(double) for each retry.
</description>
</property>
<property>
<name>dfs.ha.zkfc.nn.http.timeout.ms</name>
<value>20000</value>
<description>
The HTTP connection and read timeout value (unit is ms ) when DFS ZKFC
tries to get local NN thread dump after local NN becomes
SERVICE_NOT_RESPONDING or SERVICE_UNHEALTHY.
If it is set to zero, DFS ZKFC won't get local NN thread dump.
</description>
</property>
<property>
<name>dfs.ha.tail-edits.in-progress</name>
<value>false</value>
<description>
Whether enable standby namenode to tail in-progress edit logs.
Clients might want to turn it on when they want Standby NN to have
more up-to-date data.
</description>
</property>
<property>
<name>dfs.namenode.ec.system.default.policy</name>
<value>RS-6-3-1024k</value>
<description>The default erasure coding policy name will be used
on the path if no policy name is passed.
</description>
</property>
<property>
<name>dfs.namenode.ec.policies.max.cellsize</name>
<value>4194304</value>
<description>The maximum cell size of erasure coding policy. Default is 4MB.
</description>
</property>
<property>
<name>dfs.datanode.ec.reconstruction.stripedread.timeout.millis</name>
<value>5000</value>
<description>Datanode striped read timeout in milliseconds.
</description>
</property>
<property>
<name>dfs.datanode.ec.reconstruction.stripedread.buffer.size</name>
<value>65536</value>
<description>Datanode striped read buffer size.
</description>
</property>
<property>
<name>dfs.datanode.ec.reconstruction.threads</name>
<value>8</value>
<description>
Number of threads used by the Datanode for background
reconstruction work.
</description>
</property>
<property>
<name>dfs.datanode.ec.reconstruction.xmits.weight</name>
<value>0.5</value>
<description>
Datanode uses xmits weight to calculate the relative cost of EC recovery
tasks comparing to replicated block recovery, of which xmits is always 1.
Namenode then uses xmits reported from datanode to throttle recovery tasks
for EC and replicated blocks.
The xmits of an erasure coding recovery task is calculated as the maximum
value between the number of read streams and the number of write streams.
</description>
</property>
<property>
<name>dfs.namenode.quota.init-threads</name>
<value>4</value>
<description>
The number of concurrent threads to be used in quota initialization. The
speed of quota initialization also affects the namenode fail-over latency.
If the size of name space is big, try increasing this.
</description>
</property>
<property>
<name>dfs.datanode.transfer.socket.send.buffer.size</name>
<value>0</value>
<description>
Socket send buffer size for DataXceiver (mirroring packets to downstream
in pipeline). This may affect TCP connection throughput.
If it is set to zero or negative value, no buffer size will be set
explicitly, thus enable tcp auto-tuning on some system.
The default value is 0.
</description>
</property>
<property>
<name>dfs.datanode.transfer.socket.recv.buffer.size</name>
<value>0</value>
<description>
Socket receive buffer size for DataXceiver (receiving packets from client
during block writing). This may affect TCP connection throughput.
If it is set to zero or negative value, no buffer size will be set
explicitly, thus enable tcp auto-tuning on some system.
The default value is 0.
</description>
</property>
<property>
<name>dfs.namenode.upgrade.domain.factor</name>
<value>${dfs.replication}</value>
<description>
This is valid only when block placement policy is set to
BlockPlacementPolicyWithUpgradeDomain. It defines the number of
unique upgrade domains any block's replicas should have.
When the number of replicas is less or equal to this value, the policy
ensures each replica has an unique upgrade domain. When the number of
replicas is greater than this value, the policy ensures the number of
unique domains is at least this value.
</description>
</property>
<property>
<name>dfs.ha.zkfc.port</name>
<value>8019</value>
<description>
RPC port for Zookeeper Failover Controller.
</description>
</property>
<property>
<name>dfs.datanode.bp-ready.timeout</name>
<value>20s</value>
<description>
The maximum wait time for datanode to be ready before failing the
received request. Setting this to 0 fails requests right away if the
datanode is not yet registered with the namenode. This wait time
reduces initial request failures after datanode restart.
Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
</description>
</property>
<property>
<name>dfs.datanode.cached-dfsused.check.interval.ms</name>
<value>600000</value>
<description>
The interval check time of loading DU_CACHE_FILE in each volume.
When the cluster doing the rolling upgrade operations, it will
usually lead dfsUsed cache file of each volume expired and redo the
du operations in datanode and that makes datanode start slowly. Adjust
this property can make cache file be available for the time as you want.
</description>
</property>
<property>
<name>dfs.webhdfs.rest-csrf.enabled</name>
<value>false</value>
<description>
If true, then enables WebHDFS protection against cross-site request forgery
(CSRF). The WebHDFS client also uses this property to determine whether or
not it needs to send the custom CSRF prevention header in its HTTP requests.
</description>
</property>
<property>
<name>dfs.webhdfs.rest-csrf.custom-header</name>
<value>X-XSRF-HEADER</value>
<description>
The name of a custom header that HTTP requests must send when protection
against cross-site request forgery (CSRF) is enabled for WebHDFS by setting
dfs.webhdfs.rest-csrf.enabled to true. The WebHDFS client also uses this
property to determine whether or not it needs to send the custom CSRF
prevention header in its HTTP requests.
</description>
</property>
<property>
<name>dfs.webhdfs.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD,TRACE</value>
<description>
A comma-separated list of HTTP methods that do not require HTTP requests to
include a custom header when protection against cross-site request forgery
(CSRF) is enabled for WebHDFS by setting dfs.webhdfs.rest-csrf.enabled to
true. The WebHDFS client also uses this property to determine whether or
not it needs to send the custom CSRF prevention header in its HTTP requests.
</description>
</property>
<property>
<name>dfs.webhdfs.rest-csrf.browser-useragents-regex</name>
<value>^Mozilla.*,^Opera.*</value>
<description>
A comma-separated list of regular expressions used to match against an HTTP
request's User-Agent header when protection against cross-site request
forgery (CSRF) is enabled for WebHDFS by setting
dfs.webhdfs.reset-csrf.enabled to true. If the incoming User-Agent matches
any of these regular expressions, then the request is considered to be sent
by a browser, and therefore CSRF prevention is enforced. If the request's
User-Agent does not match any of these regular expressions, then the request
is considered to be sent by something other than a browser, such as scripted
automation. In this case, CSRF is not a potential attack vector, so
the prevention is not enforced. This helps achieve backwards-compatibility
with existing automation that has not been updated to send the CSRF
prevention header.
</description>
</property>
<property>
<name>dfs.xframe.enabled</name>
<value>true</value>
<description>
If true, then enables protection against clickjacking by returning
X_FRAME_OPTIONS header value set to SAMEORIGIN.
Clickjacking protection prevents an attacker from using transparent or
opaque layers to trick a user into clicking on a button
or link on another page.
</description>
</property>
<property>
<name>dfs.xframe.value</name>
<value>SAMEORIGIN</value>
<description>
This configration value allows user to specify the value for the
X-FRAME-OPTIONS. The possible values for this field are
DENY, SAMEORIGIN and ALLOW-FROM. Any other value will throw an
exception when namenode and datanodes are starting up.
</description>
</property>
<property>
<name>dfs.balancer.keytab.enabled</name>
<value>false</value>
<description>
Set to true to enable login using a keytab for Kerberized Hadoop.
</description>
</property>
<property>
<name>dfs.balancer.address</name>
<value>0.0.0.0:0</value>
<description>
The hostname used for a keytab based Kerberos login. Keytab based login
can be enabled with dfs.balancer.keytab.enabled.
</description>
</property>
<property>
<name>dfs.balancer.keytab.file</name>
<value></value>
<description>
The keytab file used by the Balancer to login as its
service principal. The principal name is configured with
dfs.balancer.kerberos.principal. Keytab based login can be
enabled with dfs.balancer.keytab.enabled.
</description>
</property>
<property>
<name>dfs.balancer.kerberos.principal</name>
<value></value>
<description>
The Balancer principal. This is typically set to
balancer/_HOST@REALM.TLD. The Balancer will substitute _HOST with its
own fully qualified hostname at startup. The _HOST placeholder
allows using the same configuration setting on different servers.
Keytab based login can be enabled with dfs.balancer.keytab.enabled.
</description>
</property>
<property>
<name>dfs.http.client.retry.policy.enabled</name>
<value>false</value>
<description>
If "true", enable the retry policy of WebHDFS client.
If "false", retry policy is turned off.
Enabling the retry policy can be quite useful while using WebHDFS to
copy large files between clusters that could timeout, or
copy files between HA clusters that could failover during the copy.
</description>
</property>
<property>
<name>dfs.http.client.retry.policy.spec</name>
<value>10000,6,60000,10</value>
<description>
Specify a policy of multiple linear random retry for WebHDFS client,
e.g. given pairs of number of retries and sleep time (n0, t0), (n1, t1),
..., the first n0 retries sleep t0 milliseconds on average,
the following n1 retries sleep t1 milliseconds on average, and so on.
</description>
</property>
<property>
<name>dfs.http.client.failover.max.attempts</name>
<value>15</value>
<description>
Specify the max number of failover attempts for WebHDFS client
in case of network exception.
</description>
</property>
<property>
<name>dfs.http.client.retry.max.attempts</name>
<value>10</value>
<description>
Specify the max number of retry attempts for WebHDFS client,
if the difference between retried attempts and failovered attempts is
larger than the max number of retry attempts, there will be no more
retries.
</description>
</property>
<property>
<name>dfs.http.client.failover.sleep.base.millis</name>
<value>500</value>
<description>
Specify the base amount of time in milliseconds upon which the
exponentially increased sleep time between retries or failovers
is calculated for WebHDFS client.
</description>
</property>
<property>
<name>dfs.http.client.failover.sleep.max.millis</name>
<value>15000</value>
<description>
Specify the upper bound of sleep time in milliseconds between
retries or failovers for WebHDFS client.
</description>
</property>
<property>
<name>dfs.namenode.hosts.provider.classname</name>
<value>org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager</value>
<description>
The class that provides access for host files.
org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager is used
by default which loads files specified by dfs.hosts and dfs.hosts.exclude.
If org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager is
used, it will load the JSON file defined in dfs.hosts.
To change class name, nn restart is required. "dfsadmin -refreshNodes" only
refreshes the configuration files used by the class.
</description>
</property>
<property>
<name>datanode.https.port</name>
<value>50475</value>
<description>
HTTPS port for DataNode.
</description>
</property>
<property>
<name>dfs.balancer.dispatcherThreads</name>
<value>200</value>
<description>
Size of the thread pool for the HDFS balancer block mover.
dispatchExecutor
</description>
</property>
<property>
<name>dfs.balancer.movedWinWidth</name>
<value>5400000</value>
<description>
Window of time in ms for the HDFS balancer tracking blocks and its
locations.
</description>
</property>
<property>
<name>dfs.balancer.moverThreads</name>
<value>1000</value>
<description>
Thread pool size for executing block moves.
moverThreadAllocator
</description>
</property>
<property>
<name>dfs.balancer.max-size-to-move</name>
<value>10737418240</value>
<description>
Maximum number of bytes that can be moved by the balancer in a single
thread.
</description>
</property>
<property>
<name>dfs.balancer.getBlocks.min-block-size</name>
<value>10485760</value>
<description>
Minimum block threshold size in bytes to ignore when fetching a source's
block list.
</description>
</property>
<property>
<name>dfs.balancer.getBlocks.size</name>
<value>2147483648</value>
<description>
Total size in bytes of Datanode blocks to get when fetching a source's
block list.
</description>
</property>
<property>
<name>dfs.balancer.block-move.timeout</name>
<value>0</value>
<description>
Maximum amount of time in milliseconds for a block to move. If this is set
greater than 0, Balancer will stop waiting for a block move completion
after this time. In typical clusters, a 3 to 5 minute timeout is reasonable.
If timeout happens to a large proportion of block moves, this needs to be
increased. It could also be that too much work is dispatched and many nodes
are constantly exceeding the bandwidth limit as a result. In that case,
other balancer parameters might need to be adjusted.
It is disabled (0) by default.
</description>
</property>
<property>
<name>dfs.balancer.max-no-move-interval</name>
<value>60000</value>
<description>
If this specified amount of time has elapsed and no block has been moved
out of a source DataNode, on more effort will be made to move blocks out of
this DataNode in the current Balancer iteration.
</description>
</property>
<property>
<name>dfs.balancer.max-iteration-time</name>
<value>1200000</value>
<description>
Maximum amount of time while an iteration can be run by the Balancer. After
this time the Balancer will stop the iteration, and reeva luate the work
needs to be done to Balance the cluster. The default value is 20 minutes.
</description>
</property>
<property>
<name>dfs.block.invalidate.limit</name>
<value>1000</value>
<description>
The maximum number of invalidate blocks sent by namenode to a datanode
per heartbeat deletion command. This property works with
"dfs.namenode.invalidate.work.pct.per.iteration" to throttle block
deletions.
</description>
</property>
<property>
<name>dfs.block.misreplication.processing.limit</name>
<value>10000</value>
<description>
Maximum number of blocks to process for initializing replication queues.
</description>
</property>
<property>
<name>dfs.block.placement.ec.classname</name>
<value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant</value>
<description>
Placement policy class for striped files.
Defaults to BlockPlacementPolicyRackFaultTolerant.class
</description>
</property>
<property>
<name>dfs.block.replicator.classname</name>
<value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault</value>
<description>
Class representing block placement policy for non-striped files.
There are four block placement policies currently being supported:
BlockPlacementPolicyDefault, BlockPlacementPolicyWithNodeGroup,
BlockPlacementPolicyRackFaultTolerant and BlockPlacementPolicyWithUpgradeDomain.
BlockPlacementPolicyDefault chooses the desired number of targets
for placing block replicas in a default way. BlockPlacementPolicyWithNodeGroup
places block replicas on environment with node-group layer. BlockPlacementPolicyRackFaultTolerant
places the replicas to more racks.
BlockPlacementPolicyWithUpgradeDomain places block replicas that honors upgrade domain policy.
The details of placing replicas are documented in the javadoc of the corresponding policy classes.
The default policy is BlockPlacementPolicyDefault, and the corresponding class is
org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault.
</description>
</property>
<property>
<name>dfs.blockreport.incremental.intervalMsec</name>
<value>0</value>
<description>
If set to a positive integer, the value in ms to wait between sending
incremental block reports from the Datanode to the Namenode.
</description>
</property>
<property>
<name>dfs.checksum.type</name>
<value>CRC32C</value>
<description>
Checksum type
</description>
</property>
<property>
<name>dfs.checksum.combine.mode</name>
<value>MD5MD5CRC</value>
<description>
Defines how lower-level chunk/block checksums are combined into file-level
checksums; the original MD5MD5CRC mode is not comparable between files
with different block layouts, while modes like COMPOSITE_CRC are
comparable independently of block layout.
</description>
</property>
<property>
<name>dfs.client.block.write.locateFollowingBlock.retries</name>
<value>5</value>
<description>
Number of retries to use when finding the next block during HDFS writes.
</description>
</property>
<property>
<name>dfs.client.failover.proxy.provider</name>
<value></value>
<description>
The prefix (plus a required nameservice ID) for the class name of the
configured Failover proxy provider for the host. For more detailed
information, please consult the "Configuration Details" section of
the HDFS High Availability documentation.
</description>
</property>
<property>
<name>dfs.client.failover.random.order</name>
<value>false</value>
<description>
Determines if the failover proxies are picked in random order instead of the
configured order. The prefix can be used with an optional nameservice ID
(of form dfs.client.failover.random.order[.nameservice]) in case multiple
nameservices exist and random order should be enabled for specific
nameservices.
</description>
</property>
<property>
<name>dfs.client.key.provider.cache.expiry</name>
<value>864000000</value>
<description>
DFS client security key cache expiration in milliseconds.
</description>
</property>
<property>
<name>dfs.client.max.block.acquire.failures</name>
<value>3</value>
<description>
Maximum failures allowed when trying to get block information from a specific datanode.
</description>
</property>
<property>
<name>dfs.client.read.prefetch.size</name>
<value></value>
<description>
The number of bytes for the DFSClient will fetch from the Namenode
during a read operation. Defaults to 10 * ${dfs.blocksize}.
</description>
</property>
<property>
<name>dfs.client.read.short.circuit.replica.stale.threshold.ms</name>
<value>1800000</value>
<description>
Threshold in milliseconds for read entries during short-circuit local reads.
</description>
</property>
<property>
<name>dfs.client.read.shortcircuit.buffer.size</name>
<value>1048576</value>
<description>
Buffer size in bytes for short-circuit local reads.
</description>
</property>
<property>
<name>dfs.client.read.striped.threadpool.size</name>
<value>18</value>
<description>
The maximum number of threads used for parallel reading
in striped layout.
</description>
</property>
<property>
<name>dfs.client.replica.accessor.builder.classes</name>
<value></value>
<description>
Comma-separated classes for building ReplicaAccessor. If the classes
are specified, client will use external BlockReader that uses the
ReplicaAccessor built by the builder.
</description>
</property>
<property>
<name>dfs.client.retry.interval-ms.get-last-block-length</name>
<value>4000</value>
<description>
Retry interval in milliseconds to wait between retries in getting
block lengths from the datanodes.
</description>
</property>
<property>
<name>dfs.client.retry.max.attempts</name>
<value>10</value>
<description>
Max retry attempts for DFSClient talking to namenodes.
</description>
</property>
<property>
<name>dfs.client.retry.policy.enabled</name>
<value>false</value>
<description>
If true, turns on DFSClient retry policy.
</description>
</property>
<property>
<name>dfs.client.retry.policy.spec</name>
<value>10000,6,60000,10</value>
<description>
Set to pairs of timeouts and retries for DFSClient.
</description>
</property>
<property>
<name>dfs.client.retry.times.get-last-block-length</name>
<value>3</value>
<description>
Number of retries for calls to fetchLocatedBlocksAndGetLastBlockLength().
</description>
</property>
<property>
<name>dfs.client.retry.window.base</name>
<value>3000</value>
<description>
Base time window in ms for DFSClient retries. For each retry attempt,
this value is extended linearly (e.g. 3000 ms for first attempt and
first retry, 6000 ms for second retry, 9000 ms for third retry, etc.).
</description>
</property>
<property>
<name>dfs.client.socket-timeout</name>
<value>60000</value>
<description>
Default timeout value in milliseconds for all sockets.
</description>
</property>
<property>
<name>dfs.client.socketcache.capacity</name>
<value>16</value>
<description>
Socket cache capacity (in entries) for short-circuit reads.
</description>
</property>
<property>
<name>dfs.client.socketcache.expiryMsec</name>
<value>3000</value>
<description>
Socket cache expiration for short-circuit reads in msec.
</description>
</property>
<property>
<name>dfs.client.test.drop.namenode.response.number</name>
<value>0</value>
<description>
The number of Namenode responses dropped by DFSClient for each RPC call. Used
for testing the NN retry cache.
</description>
</property>
<property>
<name>dfs.client.hedged.read.threadpool.size</name>
<value>0</value>
<description>
Support 'hedged' reads in DFSClient. To enable this feature, set the parameter
to a positive number. The threadpool size is how many threads to dedicate
to the running of these 'hedged', concurrent reads in your client.
</description>
</property>
<property>
<name>dfs.client.hedged.read.threshold.millis</name>
<value>500</value>
<description>
Configure 'hedged' reads in DFSClient. This is the number of milliseconds
to wait before starting up a 'hedged' read.
</description>
</property>
<property>
<name>dfs.client.write.byte-array-manager.count-limit</name>
<value>2048</value>
<description>
The maximum number of arrays allowed for each array length.
</description>
</property>
<property>
<name>dfs.client.write.byte-array-manager.count-reset-time-period-ms</name>
<value>10000</value>
<description>
The time period in milliseconds that the allocation count for each array length is
reset to zero if there is no increment.
</description>
</property>
<property>
<name>dfs.client.write.byte-array-manager.count-threshold</name>
<value>128</value>
<description>
The count threshold for each array length so that a manager is created only after the
allocation count exceeds the threshold. In other words, the particular array length
is not managed until the allocation count exceeds the threshold.
</description>
</property>
<property>
<name>dfs.client.write.byte-array-manager.enabled</name>
<value>false</value>
<description>
If true, enables byte array manager used by DFSOutputStream.
</description>
</property>
<property>
<name>dfs.client.write.max-packets-in-flight</name>
<value>80</value>
<description>
The maximum number of DFSPackets allowed in flight.
</description>
</property>
<property>
<name>dfs.content-summary.limit</name>
<value>5000</value>
<description>
The maximum content summary counts allowed in one locking period. 0 or a negative number
means no limit (i.e. no yielding).
</description>
</property>
<property>
<name>dfs.content-summary.sleep-microsec</name>
<value>500</value>
<description>
The length of time in microseconds to put the thread to sleep, between reaquiring the locks
in content summary computation.
</description>
</property>
<property>
<name>dfs.data.transfer.client.tcpnodelay</name>
<value>true</value>
<description>
If true, set TCP_NODELAY to sockets for transferring data from DFS client.
</description>
</property>
<property>
<name>dfs.data.transfer.server.tcpnodelay</name>
<value>true</value>
<description>
If true, set TCP_NODELAY to sockets for transferring data between Datanodes.
</description>
</property>
<property>
<name>dfs.datanode.balance.max.concurrent.moves</name>
<value>50</value>
<description>
Maximum number of threads for Datanode balancer pending moves. This
value is reconfigurable via the "dfsadmin -reconfig" command.
</description>
</property>
<property>
<name>dfs.datanode.fsdataset.factory</name>
<value></value>
<description>
The class name for the underlying storage that stores replicas for a
Datanode. Defaults to
org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory.
</description>
</property>
<property>
<name>dfs.datanode.fsdataset.volume.choosing.policy</name>
<value></value>
<description>
The class name of the policy for choosing volumes in the list of
directories. Defaults to
org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy.
If you would like to take into account available disk space, set the
value to
"org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy".
</description>
</property>
<property>
<name>dfs.datanode.hostname</name>
<value></value>
<description>
Optional. The hostname for the Datanode containing this
configuration file. Will be different for each machine.
Defaults to current hostname.
</description>
</property>
<property>
<name>dfs.datanode.lazywriter.interval.sec</name>
<value>60</value>
<description>
Interval in seconds for Datanodes for lazy persist writes.
</description>
</property>
<property>
<name>dfs.datanode.network.counts.cache.max.size</name>
<value>2147483647</value>
<description>
The maximum number of entries the datanode per-host network error
count cache may contain.
</description>
</property>
<property>
<name>dfs.datanode.oob.timeout-ms</name>
<value>1500,0,0,0</value>
<description>
Timeout value when sending OOB response for each OOB type, which are
OOB_RESTART, OOB_RESERVED1, OOB_RESERVED2, and OOB_RESERVED3,
respectively. Currently, only OOB_RESTART is used.
</description>
</property>
<property>
<name>dfs.datanode.parallel.volumes.load.threads.num</name>
<value></value>
<description>
Maximum number of threads to use for upgrading data directories.
The default value is the number of storage directories in the
DataNode.
</description>
</property>
<property>
<name>dfs.datanode.ram.disk.replica.tracker</name>
<value></value>
<description>
Name of the class implementing the RamDiskReplicaTracker interface.
Defaults to
org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker.
</description>
</property>
<property>
<name>dfs.datanode.restart.replica.expiration</name>
<value>50</value>
<description>
During shutdown for restart, the amount of time in seconds budgeted for
datanode restart.
</description>
</property>
<property>
<name>dfs.datanode.socket.reuse.keepalive</name>
<value>4000</value>
<description>
The window of time in ms before the DataXceiver closes a socket for a
single request. If a second request occurs within that window, the
socket can be reused.
</description>
</property>
<property>
<name>dfs.datanode.socket.write.timeout</name>
<value>480000</value>
<description>
Timeout in ms for clients socket writes to DataNodes.
</description>
</property>
<property>
<name>dfs.datanode.sync.behind.writes.in.background</name>
<value>false</value>
<description>
If set to true, then sync_file_range() system call will occur
asynchronously. This property is only valid when the property
dfs.datanode.sync.behind.writes is true.
</description>
</property>
<property>
<name>dfs.datanode.transferTo.allowed</name>
<value>true</value>
<description>
If false, break block transfers on 32-bit machines greater than
or equal to 2GB into smaller chunks.
</description>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value></value>
<description>
A list of scripts or Java classes which will be used to fence
the Active NameNode during a failover. See the HDFS High
Availability documentation for details on automatic HA
configuration.
</description>
</property>
<property>
<name>dfs.ha.standby.checkpoints</name>
<value>true</value>
<description>
If true, a NameNode in Standby state periodically takes a checkpoint
of the namespace, saves it to its local storage and then upload to
the remote NameNode.
</description>
</property>
<property>
<name>dfs.ha.zkfc.port</name>
<value>8019</value>
<description>
The port number that the zookeeper failover controller RPC
server binds to.
</description>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/tmp/hadoop/dfs/journalnode/</value>
<description>
The directory where the journal edit files are stored.
</description>
</property>
<property>
<name>dfs.journalnode.enable.sync</name>
<value>true</value>
<description>
If true, the journal nodes wil sync with each other. The journal nodes
will periodically gossip with other journal nodes to compare edit log
manifests and if they detect any missing log segment, they will download
it from the other journal nodes.
</description>
</property>
<property>
<name>dfs.journalnode.sync.interval</name>
<value>120000</value>
<description>
Time interval, in milliseconds, between two Journal Node syncs.
This configuration takes effect only if the journalnode sync is enabled
by setting the configuration parameter dfs.journalnode.enable.sync to true.
</description>
</property>
<property>
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
<value></value>
<description>
Kerberos SPNEGO principal name used by the journal node.
</description>
</property>
<property>
<name>dfs.journalnode.kerberos.principal</name>
<value></value>
<description>
Kerberos principal name for the journal node.
</description>
</property>
<property>
<name>dfs.journalnode.keytab.file</name>
<value></value>
<description>
Kerberos keytab file for the journal node.
</description>
</property>
<property>
<name>dfs.ls.limit</name>
<value>1000</value>
<description>
Limit the number of files printed by ls. If less or equal to
zero, at most DFS_LIST_LIMIT_DEFAULT (= 1000) will be printed.
</description>
</property>
<property>
<name>dfs.mover.movedWinWidth</name>
<value>5400000</value>
<description>
The minimum time interval, in milliseconds, that a block can be
moved to another location again.
</description>
</property>
<property>
<name>dfs.mover.moverThreads</name>
<value>1000</value>
<description>
Configure the balancer's mover thread pool size.
</description>
</property>
<property>
<name>dfs.mover.retry.max.attempts</name>
<value>10</value>
<description>
The maximum number of retries before the mover consider the
move failed.
</description>
</property>
<property>
<name>dfs.mover.keytab.enabled</name>
<value>false</value>
<description>
Set to true to enable login using a keytab for Kerberized Hadoop.
</description>
</property>
<property>
<name>dfs.mover.address</name>
<value>0.0.0.0:0</value>
<description>
The hostname used for a keytab based Kerberos login. Keytab based login
can be enabled with dfs.mover.keytab.enabled.
</description>
</property>
<property>
<name>dfs.mover.keytab.file</name>
<value></value>
<description>
The keytab file used by the Mover to login as its
service principal. The principal name is configured with
dfs.mover.kerberos.principal. Keytab based login can be
enabled with dfs.mover.keytab.enabled.
</description>
</property>
<property>
<name>dfs.mover.kerberos.principal</name>
<value></value>
<description>
The Mover principal. This is typically set to
mover/_HOST@REALM.TLD. The Mover will substitute _HOST with its
own fully qualified hostname at startup. The _HOST placeholder
allows using the same configuration setting on different servers.
Keytab based login can be enabled with dfs.mover.keytab.enabled.
</description>
</property>
<property>
<name>dfs.mover.max-no-move-interval</name>
<value>60000</value>
<description>
If this specified amount of time has elapsed and no block has been moved
out of a source DataNode, on more effort will be made to move blocks out of
this DataNode in the current Mover iteration.
</description>
</property>
<property>
<name>dfs.namenode.audit.log.async</name>
<value>false</value>
<description>
If true, enables asynchronous audit log.
</description>
</property>
<property>
<name>dfs.namenode.audit.log.token.tracking.id</name>
<value>false</value>
<description>
If true, adds a tracking ID for all audit log events.
</description>
</property>
<property>
<name>dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction</name>
<value>0.6</value>
<description>
Only used when the dfs.block.replicator.classname is set to
org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy.
Special value between 0 and 1, noninclusive. Increases chance of
placing blocks on Datanodes with less disk space used.
</description>
</property>
<property>
<name>dfs.namenode.backup.dnrpc-address</name>
<value></value>
<description>
Service RPC address for the backup Namenode.
</description>
</property>
<property>
<name>dfs.namenode.delegation.token.always-use</name>
<value>false</value>
<description>
For testing. Setting to true always allows the DT secret manager
to be used, even if security is disabled.
</description>
</property>
<property>
<name>dfs.namenode.edits.asynclogging</name>
<value>true</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.
</description>
</property>
<property>
<name>dfs.namenode.edits.dir.minimum</name>
<value>1</value>
<description>
dfs.namenode.edits.dir includes both required directories
(specified by dfs.namenode.edits.dir.required) and optional directories.
The number of usable optional directories must be greater than or equal
to this property. If the number of usable optional directories falls
below dfs.namenode.edits.dir.minimum, HDFS will issue an error.
This property defaults to 1.
</description>
</property>
<property>
<name>dfs.namenode.edits.journal-plugin</name>
<value></value>
<description>
When FSEditLog is creating JournalManagers from dfs.namenode.edits.dir,
and it encounters a URI with a schema different to "file" it loads the
name of the implementing class from
"dfs.namenode.edits.journal-plugin.[schema]". This class must implement
JournalManager and have a constructor which takes (Configuration, URI).
</description>
</property>
<property>
<name>dfs.namenode.file.close.num-committed-allowed</name>
<value>0</value>
<description>
Normally a file can only be closed with all its blocks are committed.
When this value is set to a positive integer N, a file can be closed
when N blocks are committed and the rest complete.
</description>
</property>
<property>
<name>dfs.namenode.inode.attributes.provider.class</name>
<value></value>
<description>
Name of class to use for delegating HDFS authorization.
</description>
</property>
<property>
<name>dfs.namenode.inode.attributes.provider.bypass.users</name>
<value></value>
<description>
A list of user principals (in secure cluster) or user names (in insecure
cluster) for whom the external attributes provider will be bypassed for all
operations. This means file attributes stored in HDFS instead of the
external provider will be used for permission checking and be returned when
requested.
</description>
</property>
<property>
<name>dfs.namenode.max-num-blocks-to-log</name>
<value>1000</value>
<description>
Puts a limit on the number of blocks printed to the log by the Namenode
after a block report.
</description>
</property>
<property>
<name>dfs.namenode.max.op.size</name>
<value>52428800</value>
<description>
Maximum opcode size in bytes.
</description>
</property>
<property>
<name>dfs.namenode.missing.checkpoint.periods.before.shutdown</name>
<value>3</value>
<description>
The number of checkpoint period windows (as defined by the property
dfs.namenode.checkpoint.period) allowed by the Namenode to perform
saving the namespace before shutdown.
</description>
</property>
<property>
<name>dfs.namenode.name.cache.threshold</name>
<value>10</value>
<description>
Frequently accessed files that are accessed more times than this
threshold are cached in the FSDirectory nameCache.
</description>
</property>
<property>
<name>dfs.namenode.replication.max-streams</name>
<value>2</value>
<description>
Hard limit for the number of highest-priority replication streams.
</description>
</property>
<property>
<name>dfs.namenode.replication.max-streams-hard-limit</name>
<value>4</value>
<description>
Hard limit for all replication streams.
</description>
</property>
<property>
<name>dfs.namenode.reconstruction.pending.timeout-sec</name>
<value>300</value>
<description>
Timeout in seconds for block reconstruction. If this value is 0 or less,
then it will default to 5 minutes.
</description>
</property>
<property>
<name>dfs.namenode.stale.datanode.minimum.interval</name>
<value>3</value>
<description>
Minimum number of missed heartbeats intervals for a datanode to
be marked stale by the Namenode. The actual interval is calculated as
(dfs.namenode.stale.datanode.minimum.interval * dfs.heartbeat.interval)
in seconds. If this value is greater than the property
dfs.namenode.stale.datanode.interval, then the calculated value above
is used.
</description>
</property>
<property>
<name>dfs.namenode.storageinfo.defragment.timeout.ms</name>
<value>4</value>
<description>
Timeout value in ms for the StorageInfo compaction run.
</description>
</property>
<property>
<name>dfs.namenode.storageinfo.defragment.interval.ms</name>
<value>600000</value>
<description>
The thread for checking the StorageInfo for defragmentation will
run periodically. The time between runs is determined by this
property.
</description>
</property>
<property>
<name>dfs.namenode.storageinfo.defragment.ratio</name>
<value>0.75</value>
<description>
The defragmentation threshold for the StorageInfo.
</description>
</property>
<property>
<name>dfs.namenode.snapshot.capture.openfiles</name>
<value>false</value>
<description>
If true, snapshots taken will have an immutable shared copy of
the open files that have valid leases. Even after the open files
grow or shrink in size, snapshot will always have the previous
point-in-time version of the open files, just like all other
closed files. Default is false.
Note: The file length captured for open files in snapshot is
whats recorded in NameNode at the time of snapshot and it may
be shorter than what the client has written till then. In order
to capture the latest length, the client can call hflush/hsync
with the flag SyncFlag.UPDATE_LENGTH on the open files handles.
</description>
</property>
<property>
<name>dfs.namenode.snapshot.skip.capture.accesstime-only-change</name>
<value>false</value>
<description>
If accessTime of a file/directory changed but there is no other
modification made to the file/directory, the changed accesstime will
not be captured in next snapshot. However, if there is other modification
made to the file/directory, the latest access time will be captured
together with the modification in next snapshot.
</description>
</property>
<property>
<name>dfs.namenode.snapshotdiff.allow.snap-root-descendant</name>
<value>true</value>
<description>
If enabled, snapshotDiff command can be run for any descendant directory
under a snapshot root directory and the diff calculation will be scoped
to the given descendant directory. Otherwise, snapshot diff command can
only be run for a snapshot root directory.
</description>
</property>
<property>
<name>dfs.namenode.snapshotdiff.listing.limit</name>
<value>1000</value>
<description>
Limit the number of entries generated by getSnapshotDiffReportListing within
one rpc call to the namenode.If less or equal to zero, at most
DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT_DEFAULT (= 1000) will be sent
across to the client within one rpc call.
</description>
</property>
<property>
<name>dfs.namenode.snapshot.max.limit</name>
<value>65536</value>
<description>
Limits the maximum number of snapshots allowed per snapshottable
directory.If the configuration is not set, the default limit
for maximum no of snapshots allowed is 65536.
</description>
</property>
<property>
<name>dfs.namenode.snapshot.skiplist.max.levels</name>
<value>0</value>
<description>
Maximum no of the skip levels to be maintained in the skip list for
storing directory snapshot diffs. By default, it is set to 0 and a linear
list will be used to store the directory snapshot diffs.
</description>
</property>
<property>
<name>dfs.namenode.snapshot.skiplist.interval</name>
<value>10</value>
<description>
The interval after which the skip levels will be formed in the skip list
for storing directory snapshot diffs. By default, value is set to 10.
</description>
</property>
<property>
<name>dfs.pipeline.ecn</name>
<value>false</value>
<description>
If true, allows ECN (explicit congestion notification) from the
Datanode.
</description>
</property>
<property>
<name>dfs.qjournal.accept-recovery.timeout.ms</name>
<value>120000</value>
<description>
Quorum timeout in milliseconds during accept phase of
recovery/synchronization for a specific segment.
</description>
</property>
<property>
<name>dfs.qjournal.finalize-segment.timeout.ms</name>
<value>120000</value>
<description>
Quorum timeout in milliseconds during finalizing for a specific
segment.
</description>
</property>
<property>
<name>dfs.qjournal.get-journal-state.timeout.ms</name>
<value>120000</value>
<description>
Timeout in milliseconds when calling getJournalState().
JournalNodes.
</description>
</property>
<property>
<name>dfs.qjournal.new-epoch.timeout.ms</name>
<value>120000</value>
<description>
Timeout in milliseconds when getting an epoch number for write
access to JournalNodes.
</description>
</property>
<property>
<name>dfs.qjournal.prepare-recovery.timeout.ms</name>
<value>120000</value>
<description>
Quorum timeout in milliseconds during preparation phase of
recovery/synchronization for a specific segment.
</description>
</property>
<property>
<name>dfs.qjournal.queued-edits.limit.mb</name>
<value>10</value>
<description>
Queue size in MB for quorum journal edits.
</description>
</property>
<property>
<name>dfs.qjournal.select-input-streams.timeout.ms</name>
<value>20000</value>
<description>
Timeout in milliseconds for accepting streams from JournalManagers.
</description>
</property>
<property>
<name>dfs.qjournal.start-segment.timeout.ms</name>
<value>20000</value>
<description>
Quorum timeout in milliseconds for starting a log segment.
</description>
</property>
<property>
<name>dfs.qjournal.write-txns.timeout.ms</name>
<value>20000</value>
<description>
Write timeout in milliseconds when writing to a quorum of remote
journals.
</description>
</property>
<property>
<name>dfs.quota.by.storage.type.enabled</name>
<value>true</value>
<description>
If true, enables quotas based on storage type.
</description>
</property>
<property>
<name>dfs.secondary.namenode.kerberos.principal</name>
<value></value>
<description>
Kerberos principal name for the Secondary NameNode.
</description>
</property>
<property>
<name>dfs.secondary.namenode.keytab.file</name>
<value></value>
<description>
Kerberos keytab file for the Secondary NameNode.
</description>
</property>
<property>
<name>dfs.web.authentication.filter</name>
<value>org.apache.hadoop.hdfs.web.AuthFilter</value>
<description>
Authentication filter class used for WebHDFS.
</description>
</property>
<property>
<name>dfs.web.authentication.simple.anonymous.allowed</name>
<value></value>
<description>
If true, allow anonymous user to access WebHDFS. Set to
false to disable anonymous authentication.
</description>
</property>
<property>
<name>dfs.web.ugi</name>
<value></value>
<description>
dfs.web.ugi is deprecated. Use hadoop.http.staticuser.user instead.
</description>
</property>
<property>
<name>dfs.webhdfs.netty.high.watermark</name>
<value>65535</value>
<description>
High watermark configuration to Netty for Datanode WebHdfs.
</description>
</property>
<property>
<name>dfs.webhdfs.netty.low.watermark</name>
<value>32768</value>
<description>
Low watermark configuration to Netty for Datanode WebHdfs.
</description>
</property>
<property>
<name>dfs.webhdfs.oauth2.access.token.provider</name>
<value></value>
<description>
Access token provider class for WebHDFS using OAuth2.
Defaults to org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider.
</description>
</property>
<property>
<name>dfs.webhdfs.oauth2.client.id</name>
<value></value>
<description>
Client id used to obtain access token with either credential or
refresh token.
</description>
</property>
<property>
<name>dfs.webhdfs.oauth2.enabled</name>
<value>false</value>
<description>
If true, enables OAuth2 in WebHDFS
</description>
</property>
<property>
<name>dfs.webhdfs.oauth2.refresh.url</name>
<value></value>
<description>
URL against which to post for obtaining bearer token with
either credential or refresh token.
</description>
</property>
<property>
<name>ssl.server.keystore.keypassword</name>
<value></value>
<description>
Keystore key password for HTTPS SSL configuration
</description>
</property>
<property>
<name>ssl.server.keystore.location</name>
<value></value>
<description>
Keystore location for HTTPS SSL configuration
</description>
</property>
<property>
<name>ssl.server.keystore.password</name>
<value></value>
<description>
Keystore password for HTTPS SSL configuration
</description>
</property>
<property>
<name>ssl.server.truststore.location</name>
<value></value>
<description>
Truststore location for HTTPS SSL configuration
</description>
</property>
<property>
<name>ssl.server.truststore.password</name>
<value></value>
<description>
Truststore password for HTTPS SSL configuration
</description>
</property>
<!--Disk baalncer properties-->
<property>
<name>dfs.disk.balancer.max.disk.throughputInMBperSec</name>
<value>10</value>
<description>Maximum disk bandwidth used by diskbalancer
during read from a source disk. The unit is MB/sec.
</description>
</property>
<property>
<name>dfs.disk.balancer.block.tolerance.percent</name>
<value>10</value>
<description>
When a disk balancer copy operation is proceeding, the datanode is still
active. So it might not be possible to move the exactly specified
amount of data. So tolerance allows us to define a percentage which
defines a good enough move.
</description>
</property>
<property>
<name>dfs.disk.balancer.max.disk.errors</name>
<value>5</value>
<description>
During a block move from a source to destination disk, we might
encounter various errors. This defines how many errors we can tolerate
before we declare a move between 2 disks (or a step) has failed.
</description>
</property>
<property>
<name>dfs.disk.balancer.plan.valid.interval</name>
<value>1d</value>
<description>
Maximum amount of time disk balancer plan is valid. This setting
supports multiple time unit suffixes as described in
dfs.heartbeat.interval. If no suffix is specified then milliseconds
is assumed.
</description>
</property>
<property>
<name>dfs.disk.balancer.enabled</name>
<value>true</value>
<description>
This enables the diskbalancer feature on a cluster. By default, disk
balancer is enabled.
</description>
</property>
<property>
<name>dfs.disk.balancer.plan.threshold.percent</name>
<value>10</value>
<description>
The percentage threshold value for volume Data Density in a plan.
If the absolute value of volume Data Density which is out of
threshold value in a node, it means that the volumes corresponding to
the disks should do the balancing in the plan. The default value is 10.
</description>
</property>
<property>
<name>dfs.namenode.provided.enabled</name>
<value>false</value>
<description>
Enables the Namenode to handle provided storages.
</description>
</property>
<property>
<name>dfs.provided.storage.id</name>
<value>DS-PROVIDED</value>
<description>
The storage ID used for provided stores.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.class</name>
<value>org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap</value>
<description>
The class that is used to specify the input format of the blocks on
provided storages. The default is
org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap which uses
file regions to describe blocks. The file regions are specified as a
delimited text file. Each file region is a 6-tuple containing the
block id, remote file path, offset into file, length of block, the
block pool id containing the block, and the generation stamp of the
block.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.inmemory.batch-size</name>
<value>500</value>
<description>
The batch size when iterating over the database backing the aliasmap
</description>
</property>
<property>
<name>dfs.provided.aliasmap.inmemory.dnrpc-address</name>
<value>0.0.0.0:50200</value>
<description>
The address where the aliasmap server will be running
</description>
</property>
<property>
<name>dfs.provided.aliasmap.inmemory.leveldb.dir</name>
<value>/tmp</value>
<description>
The directory where the leveldb files will be kept
</description>
</property>
<property>
<name>dfs.provided.aliasmap.inmemory.enabled</name>
<value>false</value>
<description>
Don't use the aliasmap by default. Some tests will fail
because they try to start the namenode twice with the
same parameters if you turn it on.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.text.delimiter</name>
<value>,</value>
<description>
The delimiter used when the provided block map is specified as
a text file.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.text.read.file</name>
<value></value>
<description>
The path specifying the provided block map as a text file, specified as
a URI.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.text.codec</name>
<value></value>
<description>
The codec used to de-compress the provided block map.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.text.write.dir</name>
<value></value>
<description>
The path to which the provided block map should be written as a text
file, specified as a URI.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.leveldb.path</name>
<value></value>
<description>
The read/write path for the leveldb-based alias map
(org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap).
The path has to be explicitly configured when this alias map is used.
</description>
</property>
<property>
<name>dfs.provided.aliasmap.load.retries</name>
<value>0</value>
<description>
The number of retries on the Datanode to load the provided aliasmap;
defaults to 0.
</description>
</property>
<property>
<name>dfs.lock.suppress.warning.interval</name>
<value>10s</value>
<description>Instrumentation reporting long critical sections will suppress
consecutive warnings within this interval.</description>
</property>
<property>
<name>httpfs.buffer.size</name>
<value>4096</value>
<description>
The size buffer to be used when creating or opening httpfs filesystem IO stream.
</description>
</property>
<property>
<name>dfs.webhdfs.use.ipc.callq</name>
<value>true</value>
<description>Enables routing of webhdfs calls through rpc
call queue</description>
</property>
<property>
<name>dfs.datanode.disk.check.min.gap</name>
<value>15m</value>
<description>
The minimum gap between two successive checks of the same DataNode
volume. This setting supports multiple time unit suffixes as described
in dfs.heartbeat.interval. If no suffix is specified then milliseconds
is assumed.
</description>
</property>
<property>
<name>dfs.datanode.disk.check.timeout</name>
<value>10m</value>
<description>
Maximum allowed time for a disk check to complete during DataNode
startup. If the check does not complete within this time interval
then the disk is declared as failed. This setting supports
multiple time unit suffixes as described in dfs.heartbeat.interval.
If no suffix is specified then milliseconds is assumed.
</description>
</property>
<property>
<name>dfs.use.dfs.network.topology</name>
<value>true</value>
<description>
Enables DFSNetworkTopology to choose nodes for placing replicas.
When enabled, NetworkTopology will be instantiated as class defined in
property dfs.net.topology.impl, otherwise NetworkTopology will be
instantiated as class defined in property net.topology.impl.
</description>
</property>
<property>
<name>dfs.net.topology.impl</name>
<value>org.apache.hadoop.hdfs.net.DFSNetworkTopology</value>
<description>
The implementation class of NetworkTopology used in HDFS. By default,
the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and
used in block placement.
This property only works when dfs.use.dfs.network.topology is true.
</description>
</property>
<property>
<name>dfs.qjm.operations.timeout</name>
<value>60s</value>
<description>
Common key to set timeout for related operations in
QuorumJournalManager. This setting supports multiple time unit suffixes
as described in dfs.heartbeat.interval.
If no suffix is specified then milliseconds is assumed.
</description>
</property>
<property>
<name>dfs.reformat.disabled</name>
<value>false</value>
<description>
Disable reformat of NameNode. If it's value is set to "true"
and metadata directories already exist then attempt to format NameNode
will throw NameNodeFormatException.
</description>
</property>
</configuration>
3.mapred-default.xml
<xml version="1.0">
<xml-stylesheet type="text/xsl" href="configuration.xsl">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Do not modify this file directly. Instead, copy entries that you -->
<!-- wish to modify from this file into mapred-site.xml and change them -->
<!-- there. If mapred-site.xml does not already exist, create it. -->
<configuration>
<property>
<name>mapreduce.job.hdfs-servers</name>
<value>${fs.defaultFS}</value>
</property>
<property>
<name>mapreduce.job.committer.setup.cleanup.needed</name>
<value>true</value>
<description> true, if job needs job-setup and job-cleanup.
false, otherwise
</description>
</property>
<!-- i/o properties -->
<property>
<name>mapreduce.task.io.sort.factor</name>
<value>10</value>
<description>The number of streams to merge at once while sorting
files. This determines the number of open file handles.</description>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
<value>100</value>
<description>The total amount of buffer memory to use while sorting
files, in megabytes. By default, gives each merge stream 1MB, which
should minimize seeks.</description>
</property>
<property>
<name>mapreduce.map.sort.spill.percent</name>
<value>0.80</value>
<description>The soft limit in the serialization buffer. Once reached, a
thread will begin to spill the contents to disk in the background. Note that
collection will not block if this threshold is exceeded while a spill is
already in progress, so spills may be larger than this threshold when it is
set to less than .5</description>
</property>
<property>
<name>mapreduce.job.local-fs.single-disk-limit.bytes</name>
<value>-1</value>
<description>Enable an in task monitor thread to watch for single disk
consumption by jobs. By setting this to x nr of bytes, the task will fast
fail in case it is reached. This is a per disk configuration.</description>
</property>
<property>
<name>mapreduce.job.local-fs.single-disk-limit.check.interval-ms</name>
<value>5000</value>
<description>Interval of disk limit check to run in ms.</description>
</property>
<property>
<name>mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed</name>
<value>true</value>
<description>If mapreduce.job.local-fs.single-disk-limit.bytes is triggered
should the task be killed or logged. If false the intent to kill the task
is only logged in the container logs.</description>
</property>
<property>
<name>mapreduce.job.maps</name>
<value>2</value>
<description>The default number of map tasks per job.
Ignored when mapreduce.framework.name is "local".
</description>
</property>
<property>
<name>mapreduce.job.reduces</name>
<value>1</value>
<description>The default number of reduce tasks per job. Typically set to 99%
of the cluster's reduce capacity, so that if a node fails the reduces can
still be executed in a single wave.
Ignored when mapreduce.framework.name is "local".
</description>
</property>
<property>
<name>mapreduce.job.running.map.limit</name>
<value>0</value>
<description>The maximum number of simultaneous map tasks per job.
There is no limit if this value is 0 or negative.
</description>
</property>
<property>
<name>mapreduce.job.running.reduce.limit</name>
<value>0</value>
<description>The maximum number of simultaneous reduce tasks per job.
There is no limit if this value is 0 or negative.
</description>
</property>
<property>
<name>mapreduce.job.max.map</name>
<value>-1</value>
<description>Limit on the number of map tasks allowed per job.
There is no limit if this value is negative.
</description>
</property>
<property>
<name>mapreduce.job.reducer.preempt.delay.sec</name>
<value>0</value>
<description>The threshold (in seconds) after which an unsatisfied
mapper request triggers reducer preemption when there is no anticipated
headroom. If set to 0 or a negative value, the reducer is preempted as
soon as lack of headroom is detected. Default is 0.
</description>
</property>
<property>
<name>mapreduce.job.reducer.unconditional-preempt.delay.sec</name>
<value>300</value>
<description>The threshold (in seconds) after which an unsatisfied
mapper request triggers a forced reducer preemption irrespective of the
anticipated headroom. By default, it is set to 5 mins. Setting it to 0
leads to immediate reducer preemption. Setting to -1 disables this
preemption altogether.
</description>
</property>
<property>
<name>mapreduce.job.max.split.locations</name>
<value>10</value>
<description>The max number of block locations to store for each split for
locality calculation.
</description>
</property>
<property>
<name>mapreduce.job.split.metainfo.maxsize</name>
<value>10000000</value>
<description>The maximum permissible size of the split metainfo file.
The MapReduce ApplicationMaster won't attempt to read submitted split metainfo
files bigger than this configured value.
No limits if set to -1.
</description>
</property>
<property>
<name>mapreduce.map.maxattempts</name>
<value>4</value>
<description>Expert: The maximum number of attempts per map task.
In other words, framework will try to execute a map task these many number
of times before giving up on it.
</description>
</property>
<property>
<name>mapreduce.reduce.maxattempts</name>
<value>4</value>
<description>Expert: The maximum number of attempts per reduce task.
In other words, framework will try to execute a reduce task these many number
of times before giving up on it.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
<value>${yarn.nodemanager.recovery.enabled}</value>
<description>Set to enable fetch retry during host restart.</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
<value>1000</value>
<description>Time of interval that fetcher retry to fetch again when some
non-fatal failure happens because of some events like NM restart.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
<value>30000</value>
<description>Timeout value for fetcher to retry to fetch again when some
non-fatal failure happens because of some events like NM restart.</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.retry-delay.max.ms</name>
<value>60000</value>
<description>The maximum number of ms the reducer will delay before retrying
to download map data.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>5</value>
<description>The default number of parallel transfers run by reduce
during the copy(shuffle) phase.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.connect.timeout</name>
<value>180000</value>
<description>Expert: The maximum amount of time (in milli seconds) reduce
task spends in trying to connect to a remote node for getting map output.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.read.timeout</name>
<value>180000</value>
<description>Expert: The maximum amount of time (in milli seconds) reduce
task waits for map output data to be available for reading after obtaining
connection.
</description>
</property>
<property>
<name>mapreduce.shuffle.listen.queue.size</name>
<value>128</value>
<description>The length of the shuffle server listen queue.</description>
</property>
<property>
<name>mapreduce.shuffle.connection-keep-alive.enable</name>
<value>false</value>
<description>set to true to support keep-alive connections.</description>
</property>
<property>
<name>mapreduce.shuffle.connection-keep-alive.timeout</name>
<value>5</value>
<description>The number of seconds a shuffle client attempts to retain
http connection. Refer "Keep-Alive: timeout=" header in
Http specification
</description>
</property>
<property>
<name>mapreduce.task.timeout</name>
<value>600000</value>
<description>The number of milliseconds before a task will be
terminated if it neither reads an input, writes an output, nor
updates its status string. A value of 0 disables the timeout.
</description>
</property>
<property>
<name>mapreduce.map.memory.mb</name>
<value>-1</value>
<description>The amount of memory to request from the scheduler for each
map task. If this is not specified or is non-positive, it is inferred from
mapreduce.map.java.opts and mapreduce.job.heap.memory-mb.ratio.
If java-opts are also not specified, we set it to 1024.
</description>
</property>
<property>
<name>mapreduce.map.cpu.vcores</name>
<value>1</value>
<description>The number of virtual cores to request from the scheduler for
each map task.
</description>
</property>
<property>
<name>mapreduce.reduce.memory.mb</name>
<value>-1</value>
<description>The amount of memory to request from the scheduler for each
reduce task. If this is not specified or is non-positive, it is inferred
from mapreduce.reduce.java.opts and mapreduce.job.heap.memory-mb.ratio.
If java-opts are also not specified, we set it to 1024.
</description>
</property>
<property>
<name>mapreduce.reduce.cpu.vcores</name>
<value>1</value>
<description>The number of virtual cores to request from the scheduler for
each reduce task.
</description>
</property>
<property>
<name>mapred.child.java.opts</name>
<value></value>
<description>Java opts for the task processes.
The following symbol, if present, will be interpolated: @taskid@ is replaced
by current TaskID. Any other occurrences of '@' will go unchanged.
For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
Usage of -Djava.library.path can cause programs to no longer function if
hadoop native libraries are used. These values should instead be set as part
of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
mapreduce.reduce.env config settings.
If -Xmx is not set, it is inferred from mapreduce.{map|reduce}.memory.mb and
mapreduce.job.heap.memory-mb.ratio.
</description>
</property>
<!-- This is commented out so that it won't override mapred.child.java.opts.
<property>
<name>mapreduce.map.java.opts</name>
<value></value>
<description>Java opts only for the child processes that are maps. If set,
this will be used instead of mapred.child.java.opts. If -Xmx is not set,
it is inferred from mapreduce.map.memory.mb and
mapreduce.job.heap.memory-mb.ratio.
</description>
</property>
-->
<!-- This is commented out so that it won't override mapred.child.java.opts.
<property>
<name>mapreduce.reduce.java.opts</name>
<value></value>
<description>Java opts only for the child processes that are reduces. If set,
this will be used instead of mapred.child.java.opts. If -Xmx is not set,
it is inferred from mapreduce.reduce.memory.mb and
mapreduce.job.heap.memory-mb.ratio.
</description>
</property>
-->
<property>
<name>mapred.child.env</name>
<value></value>
<description>User added environment variables for the task processes.
Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit nodemanager's B env variable on Unix.
3) B=%B%;c This is inherit nodemanager's B env variable on Windows.
</description>
</property>
<!-- This is commented out so that it won't override mapred.child.env.
<property>
<name>mapreduce.map.env</name>
<value></value>
<description>User added environment variables for the map task processes.
</description>
</property>
-->
<!-- This is commented out so that it won't override mapred.child.env.
<property>
<name>mapreduce.reduce.env</name>
<value></value>
<description>User added environment variables for the reduce task processes.
</description>
</property>
-->
<property>
<name>mapreduce.admin.user.env</name>
<value></value>
<description>
Expert: Additional execution environment entries for
map and reduce task processes. This is not an additive property.
You must preserve the original value if you want your map and
reduce tasks to have access to native libraries (compression, etc).
When this value is empty, the command to set execution
envrionment will be OS dependent:
For linux, use LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native.
For windows, use PATH = %PATH%;%HADOOP_COMMON_HOME%\\bin.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.log.level</name>
<value>INFO</value>
<description>The logging level for the MR ApplicationMaster. The allowed
levels are: OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
The setting here could be overriden if "mapreduce.job.log4j-properties-file"
is set.
</description>
</property>
<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>
<description>The logging level for the map task. The allowed levels are:
OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
The setting here could be overridden if "mapreduce.job.log4j-properties-file"
is set.
</description>
</property>
<property>
<name>mapreduce.reduce.log.level</name>
<value>INFO</value>
<description>The logging level for the reduce task. The allowed levels are:
OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
The setting here could be overridden if "mapreduce.job.log4j-properties-file"
is set.
</description>
</property>
<property>
<name>mapreduce.reduce.merge.inmem.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
for the in-memory merge process. When we accumulate threshold number of files
we initiate the in-memory merge and spill to disk. A value of 0 or less than
0 indicates we want to DON'T have any threshold and instead depend only on
the ramfs's memory consumption to trigger the merge.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.merge.percent</name>
<value>0.66</value>
<description>The usage threshold at which an in-memory merge will be
initiated, expressed as a percentage of the total memory allocated to
storing in-memory map outputs, as defined by
mapreduce.reduce.shuffle.input.buffer.percent.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.input.buffer.percent</name>
<value>0.70</value>
<description>The percentage of memory to be allocated from the maximum heap
size to storing map outputs during the shuffle.
</description>
</property>
<property>
<name>mapreduce.reduce.input.buffer.percent</name>
<value>0.0</value>
<description>The percentage of memory- relative to the maximum heap size- to
retain map outputs during the reduce. When the shuffle is concluded, any
remaining map outputs in memory must consume less than this threshold before
the reduce can begin.
</description>
</property>
<property>
<name>mapreduce.reduce.shuffle.memory.limit.percent</name>
<value>0.25</value>
<description>Expert: Maximum percentage of the in-memory limit that a
single shuffle can consume. Range of valid values is [0.0, 1.0]. If the value
is 0.0 map outputs are shuffled directly to disk.</description>
</property>
<property>
<name>mapreduce.shuffle.ssl.enabled</name>
<value>false</value>
<description>
Whether to use SSL for for the Shuffle HTTP endpoints.
</description>
</property>
<property>
<name>mapreduce.shuffle.ssl.file.buffer.size</name>
<value>65536</value>
<description>Buffer size for reading spills from file when using SSL.
</description>
</property>
<property>
<name>mapreduce.shuffle.max.connections</name>
<value>0</value>
<description>Max allowed connections for the shuffle. Set to 0 (zero)
to indicate no limit on the number of connections.
</description>
</property>
<property>
<name>mapreduce.shuffle.max.threads</name>
<value>0</value>
<description>Max allowed threads for serving shuffle connections. Set to zero
to indicate the default of 2 times the number of available
processors (as reported by Runtime.availableProcessors()). Netty is used to
serve requests, so a thread is not needed for each connection.
</description>
</property>
<property>
<name>mapreduce.shuffle.transferTo.allowed</name>
<value></value>
<description>This option can enable/disable using nio transferTo method in
the shuffle phase. NIO transferTo does not perform well on windows in the
shuffle phase. Thus, with this configuration property it is possible to
disable it, in which case custom transfer method will be used. Recommended
value is false when running Hadoop on Windows. For Linux, it is recommended
to set it to true. If nothing is set then the default value is false for
Windows, and true for Linux.
</description>
</property>
<property>
<name>mapreduce.shuffle.transfer.buffer.size</name>
<value>131072</value>
<description>This property is used only if
mapreduce.shuffle.transferTo.allowed is set to false. In that case,
this property defines the size of the buffer used in the buffer copy code
for the shuffle phase. The size of this buffer determines the size of the IO
requests.
</description>
</property>
<property>
<name>mapreduce.reduce.markreset.buffer.percent</name>
<value>0.0</value>
<description>The percentage of memory -relative to the maximum heap size- to
be used for caching values when using the mark-reset functionality.
</description>
</property>
<property>
<name>mapreduce.map.speculative</name>
<value>true</value>
<description>If true, then multiple instances of some map tasks
may be executed in parallel.</description>
</property>
<property>
<name>mapreduce.reduce.speculative</name>
<value>true</value>
<description>If true, then multiple instances of some reduce tasks
may be executed in parallel.</description>
</property>
<property>
<name>mapreduce.job.speculative.speculative-cap-running-tasks</name>
<value>0.1</value>
<description>The max percent (0-1) of running tasks that
can be speculatively re-executed at any time.</description>
</property>
<property>
<name>mapreduce.job.speculative.speculative-cap-total-tasks</name>
<value>0.01</value>
<description>The max percent (0-1) of all tasks that
can be speculatively re-executed at any time.</description>
</property>
<property>
<name>mapreduce.job.speculative.minimum-allowed-tasks</name>
<value>10</value>
<description>The minimum allowed tasks that
can be speculatively re-executed at any time.</description>
</property>
<property>
<name>mapreduce.job.speculative.retry-after-no-speculate</name>
<value>1000</value>
<description>The waiting time(ms) to do next round of speculation
if there is no task speculated in this round.</description>
</property>
<property>
<name>mapreduce.job.speculative.retry-after-speculate</name>
<value>15000</value>
<description>The waiting time(ms) to do next round of speculation
if there are tasks speculated in this round.</description>
</property>
<property>
<name>mapreduce.job.map.output.collector.class</name>
<value>org.apache.hadoop.mapred.MapTask$MapOutputBuffer</value>
<description>
The MapOutputCollector implementation(s) to use. This may be a comma-separated
list of class names, in which case the map task will try to initialize each
of the collectors in turn. The first to successfully initialize will be used.
</description>
</property>
<property>
<name>mapreduce.job.speculative.slowtaskthreshold</name>
<value>1.0</value>
<description>The number of standard deviations by which a task's
ave progress-rates must be lower than the average of all running tasks'
for the task to be considered too slow.
</description>
</property>
<property>
<name>mapreduce.job.ubertask.enable</name>
<value>false</value>
<description>Whether to enable the small-jobs "ubertask" optimization,
which runs "sufficiently small" jobs sequentially within a single JVM.
"Small" is defined by the following maxmaps, maxreduces, and maxbytes
settings. Note that configurations for application masters also affect
the "Small" definition - yarn.app.mapreduce.am.resource.mb must be
larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb,
and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than
both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable
ubertask. Users may override this value.
</description>
</property>
<property>
<name>mapreduce.job.ubertask.maxmaps</name>
<value>9</value>
<description>Threshold for number of maps, beyond which job is considered
too big for the ubertasking optimization. Users may override this value,
but only downward.
</description>
</property>
<property>
<name>mapreduce.job.ubertask.maxreduces</name>
<value>1</value>
<description>Threshold for number of reduces, beyond which job is considered
too big for the ubertasking optimization. CURRENTLY THE CODE CANNOT SUPPORT
MORE THAN ONE REDUCE and will ignore larger values. (Zero is a valid max,
however.) Users may override this value, but only downward.
</description>
</property>
<property>
<name>mapreduce.job.ubertask.maxbytes</name>
<value></value>
<description>Threshold for number of input bytes, beyond which job is
considered too big for the ubertasking optimization. If no value is
specified, dfs.block.size is used as a default. Be sure to specify a
default value in mapred-site.xml if the underlying filesystem is not HDFS.
Users may override this value, but only downward.
</description>
</property>
<property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
<description>Specifies if the Application Master should emit timeline data
to the timeline server. Individual jobs can override this value.
</description>
</property>
<property>
<name>mapreduce.job.sharedcache.mode</name>
<value>disabled</value>
<description>
A comma delimited list of resource categories to submit to the shared cache.
The valid categories are: jobjar, libjars, files, archives.
If "disabled" is specified then the job submission code will not use
the shared cache.
</description>
</property>
<property>
<name>mapreduce.input.fileinputformat.split.minsize</name>
<value>0</value>
<description>The minimum size chunk that map input should be split
into. Note that some file formats may have minimum split sizes that
take priority over this setting.</description>
</property>
<property>
<name>mapreduce.input.fileinputformat.list-status.num-threads</name>
<value>1</value>
<description>The number of threads to use to list and fetch block locations
for the specified input paths. Note: multiple threads should not be used
if a custom non thread-safe path filter is used.
</description>
</property>
<property>
<name>mapreduce.input.lineinputformat.linespermap</name>
<value>1</value>
<description>When using NLineInputFormat, the number of lines of input data
to include in each split.</description>
</property>
<property>
<name>mapreduce.client.submit.file.replication</name>
<value>10</value>
<description>The replication level for submitted job files. This
should be around the square root of the number of nodes.
</description>
</property>
<property>
<name>mapreduce.task.files.preserve.failedtasks</name>
<value>false</value>
<description>Should the files for failed tasks be kept. This should only be
used on jobs that are failing, because the storage is never
reclaimed. It also prevents the map outputs from being erased
from the reduce directory as they are consumed.</description>
</property>
<!--
<property>
<name>mapreduce.task.files.preserve.filepattern</name>
<value>.*_m_123456_0</value>
<description>Keep all files from tasks whose task names match the given
regular expression. Defaults to none.</description>
</property>
-->
<property>
<name>mapreduce.output.fileoutputformat.compress</name>
<value>false</value>
<description>Should the job outputs be compressed
</description>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.type</name>
<value>RECORD</value>
<description>If the job outputs are to compressed as SequenceFiles, how should
they be compressed Should be one of NONE, RECORD or BLOCK.
</description>
</property>
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
<description>If the job outputs are compressed, how should they be compressed
</description>
</property>
<property>
<name>mapreduce.map.output.compress</name>
<value>false</value>
<description>Should the outputs of the maps be compressed before being
sent across the network. Uses SequenceFile compression.
</description>
</property>
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.DefaultCodec</value>
<description>If the map outputs are compressed, how should they be
compressed
</description>
</property>
<property>
<name>map.sort.class</name>
<value>org.apache.hadoop.util.QuickSort</value>
<description>The default sort class for sorting keys.
</description>
</property>
<property>
<name>mapreduce.task.userlog.limit.kb</name>
<value>0</value>
<description>The maximum size of user-logs of each task in KB. 0 disables the cap.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.container.log.limit.kb</name>
<value>0</value>
<description>The maximum size of the MRAppMaster attempt container logs in KB.
0 disables the cap.
</description>
</property>
<property>
<name>yarn.app.mapreduce.task.container.log.backups</name>
<value>0</value>
<description>Number of backup files for task logs when using
ContainerRollingLogAppender (CRLA). See
org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
is enabled for tasks when both mapreduce.task.userlog.limit.kb and
yarn.app.mapreduce.task.container.log.backups are greater than zero.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.container.log.backups</name>
<value>0</value>
<description>Number of backup files for the ApplicationMaster logs when using
ContainerRollingLogAppender (CRLA). See
org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
is enabled for the ApplicationMaster when both
yarn.app.mapreduce.am.container.log.limit.kb and
yarn.app.mapreduce.am.container.log.backups are greater than zero.
</description>
</property>
<property>
<name>yarn.app.mapreduce.shuffle.log.separate</name>
<value>true</value>
<description>If enabled ('true') logging generated by the client-side shuffle
classes in a reducer will be written in a dedicated log file
'syslog.shuffle' instead of 'syslog'.
</description>
</property>
<property>
<name>yarn.app.mapreduce.shuffle.log.limit.kb</name>
<value>0</value>
<description>Maximum size of the syslog.shuffle file in kilobytes
(0 for no limit).
</description>
</property>
<property>
<name>yarn.app.mapreduce.shuffle.log.backups</name>
<value>0</value>
<description>If yarn.app.mapreduce.shuffle.log.limit.kb and
yarn.app.mapreduce.shuffle.log.backups are greater than zero
then a ContainerRollngLogAppender is used instead of ContainerLogAppender
for syslog.shuffle. See
org.apache.log4j.RollingFileAppender.maxBackupIndex
</description>
</property>
<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
<value>3</value>
<description>The number of task-failures on a node manager of a given job
after which new tasks of that job aren't assigned to it. It
MUST be less than mapreduce.map.maxattempts and
mapreduce.reduce.maxattempts otherwise the failed task will
never be tried on a different node.
</description>
</property>
<property>
<name>mapreduce.client.output.filter</name>
<value>FAILED</value>
<description>The filter for controlling the output of the task's userlogs sent
to the console of the JobClient.
The permissible options are: NONE, KILLED, FAILED, SUCCEEDED and
ALL.
</description>
</property>
<property>
<name>mapreduce.client.completion.pollinterval</name>
<value>5000</value>
<description>The interval (in milliseconds) between which the JobClient
polls the MapReduce ApplicationMaster for updates about job status. You may want to
set this to a lower value to make tests run faster on a single node system. Adjusting
this value in production may lead to unwanted client-server traffic.
</description>
</property>
<property>
<name>mapreduce.client.progressmonitor.pollinterval</name>
<value>1000</value>
<description>The interval (in milliseconds) between which the JobClient
reports status to the console and checks for job completion. You may want to set this
to a lower value to make tests run faster on a single node system. Adjusting
this value in production may lead to unwanted client-server traffic.
</description>
</property>
<property>
<name>mapreduce.client.libjars.wildcard</name>
<value>true</value>
<description>
Whether the libjars cache files should be localized using
a wildcarded directory instead of naming each archive independently.
Using wildcards reduces the space needed for storing the job
information in the case of a highly available resource manager
configuration.
This propery should only be set to false for specific
jobs which are highly sensitive to the details of the archive
localization. Having this property set to true will cause the archives
to all be localized to the same local cache location. If false, each
archive will be localized to its own local cache location. In both
cases a symbolic link will be created to every archive from the job's
working directory.
</description>
</property>
<property>
<name>mapreduce.task.profile</name>
<value>false</value>
<description>To set whether the system should collect profiler
information for some of the tasks in this job The information is stored
in the user log directory. The value is "true" if task profiling
is enabled.</description>
</property>
<property>
<name>mapreduce.task.profile.maps</name>
<value>0-2</value>
<description> To set the ranges of map tasks to profile.
mapreduce.task.profile has to be set to true for the value to be accounted.
</description>
</property>
<property>
<name>mapreduce.task.profile.reduces</name>
<value>0-2</value>
<description> To set the ranges of reduce tasks to profile.
mapreduce.task.profile has to be set to true for the value to be accounted.
</description>
</property>
<property>
<name>mapreduce.task.profile.params</name>
<value>-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s</value>
<description>JVM profiler parameters used to profile map and reduce task
attempts. This string may contain a single format specifier %s that will
be replaced by the path to profile.out in the task attempt log directory.
To specify different profiling options for map tasks and reduce tasks,
more specific parameters mapreduce.task.profile.map.params and
mapreduce.task.profile.reduce.params should be used.</description>
</property>
<property>
<name>mapreduce.task.profile.map.params</name>
<value>${mapreduce.task.profile.params}</value>
<description>Map-task-specific JVM profiler parameters. See
mapreduce.task.profile.params</description>
</property>
<property>
<name>mapreduce.task.profile.reduce.params</name>
<value>${mapreduce.task.profile.params}</value>
<description>Reduce-task-specific JVM profiler parameters. See
mapreduce.task.profile.params</description>
</property>
<property>
<name>mapreduce.task.skip.start.attempts</name>
<value>2</value>
<description> The number of Task attempts AFTER which skip mode
will be kicked off. When skip mode is kicked off, the
tasks reports the range of records which it will process
next, to the MR ApplicationMaster. So that on failures, the MR AM
knows which ones are possibly the bad records. On further executions,
those are skipped.
</description>
</property>
<property>
<name>mapreduce.job.skip.outdir</name>
<value></value>
<description> If no value is specified here, the skipped records are
written to the output directory at _logs/skip.
User can stop writing skipped records by giving the value "none".
</description>
</property>
<property>
<name>mapreduce.map.skip.maxrecords</name>
<value>0</value>
<description> The number of acceptable skip records surrounding the bad
record PER bad record in mapper. The number includes the bad record as well.
To turn the feature of detection/skipping of bad records off, set the
value to 0.
The framework tries to narrow down the skipped range by retrying
until this threshold is met OR all attempts get exhausted for this task.
Set the value to Long.MAX_VALUE to indicate that framework need not try to
narrow down. Whatever records(depends on application) get skipped are
acceptable.
</description>
</property>
<property>
<name>mapreduce.map.skip.proc-count.auto-incr</name>
<value>true</value>
<description>The flag which if set to true,
SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented by
MapRunner after invoking the map function. This value must be set
to false for applications which process the records asynchronously
or buffer the input records. For example streaming. In such cases
applications should increment this counter on their own.
</description>
</property>
<property>
<name>mapreduce.reduce.skip.maxgroups</name>
<value>0</value>
<description> The number of acceptable skip groups surrounding the bad
group PER bad group in reducer. The number includes the bad group as well.
To turn the feature of detection/skipping of bad groups off, set the
value to 0.
The framework tries to narrow down the skipped range by retrying
until this threshold is met OR all attempts get exhausted for this task.
Set the value to Long.MAX_VALUE to indicate that framework need not try to
narrow down. Whatever groups(depends on application) get skipped are
acceptable.
</description>
</property>
<property>
<name>mapreduce.reduce.skip.proc-count.auto-incr</name>
<value>true</value>
<description>The flag which if set to true.
SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented by framework
after invoking the reduce function. This value must be set to false for
applications which process the records asynchronously or buffer the input
records. For example streaming. In such cases applications should increment
this counter on their own.
</description>
</property>
<property>
<name>mapreduce.ifile.readahead</name>
<value>true</value>
<description>Configuration key to enable/disable IFile readahead.
</description>
</property>
<property>
<name>mapreduce.ifile.readahead.bytes</name>
<value>4194304</value>
<description>Configuration key to set the IFile readahead length in bytes.
</description>
</property>
<property>
<name>mapreduce.job.queuename</name>
<value>default</value>
<description> Queue to which a job is submitted. This must match one of the
queues defined in mapred-queues.xml for the system. Also, the ACL setup
for the queue must allow the current user to submit a job to the queue.
Before specifying a queue, ensure that the system is configured with
the queue, and access is allowed for submitting jobs to the queue.
</description>
</property>
<property>
<name>mapreduce.job.tags</name>
<value></value>
<description> Tags for the job that will be passed to YARN at submission
time. Queries to YARN for applications can filter on these tags.
If these tags are intended to be used with The YARN Timeline Service v.2,
prefix them with the appropriate tag names for flow name, flow version and
flow run id. Example:
timeline_flow_name_tag:foo,
timeline_flow_version_tag:3df8b0d6100530080d2e0decf9e528e57c42a90a,
timeline_flow_run_id_tag:1465246348599
</description>
</property>
<property>
<name>mapreduce.cluster.local.dir</name>
<value>${hadoop.tmp.dir}/mapred/local</value>
<description>
The local directory where MapReduce stores intermediate
data files. May be a comma-separated list of
directories on different devices in order to spread disk i/o.
Directories that do not exist are ignored.
</description>
</property>
<property>
<name>mapreduce.cluster.acls.enabled</name>
<value>false</value>
<description> Specifies whether ACLs should be checked
for authorization of users for doing various queue and job level operations.
ACLs are disabled by default. If enabled, access control checks are made by
MapReduce ApplicationMaster when requests are made by users for queue
operations like submit job to a queue and kill a job in the queue and job
operations like viewing the job-details (See mapreduce.job.acl-view-job)
or for modifying the job (See mapreduce.job.acl-modify-job) using
Map/Reduce APIs, RPCs or via the console and web user interfaces.
For enabling this flag, set to true in mapred-site.xml file of all
MapReduce clients (MR job submitting nodes).
</description>
</property>
<property>
<name>mapreduce.job.acl-modify-job</name>
<value> </value>
<description> Job specific access-control list for 'modifying' the job. It
is only used if authorization is enabled in Map/Reduce by setting the
configuration property mapreduce.cluster.acls.enabled to true.
This specifies the list of users and/or groups who can do modification
operations on the job. For specifying a list of users and groups the
format to use is "user1,user2 group1,group". If set to '*', it allows all
users/groups to modify this job. If set to ' '(i.e. space), it allows
none. This configuration is used to guard all the modifications with respect
to this job and takes care of all the following operations:
o killing this job
o killing a task of this job, failing a task of this job
o setting the priority of this job
Each of these operations are also protected by the per-queue level ACL
"acl-administer-jobs" configured via mapred-queues.xml. So a caller should
have the authorization to satisfy either the queue-level ACL or the
job-level ACL.
Irrespective of this ACL configuration, (a) job-owner, (b) the user who
started the cluster, (c) members of an admin configured supergroup
configured via mapreduce.cluster.permissions.supergroup and (d) queue
administrators of the queue to which this job was submitted to configured
via acl-administer-jobs for the specific queue in mapred-queues.xml can
do all the modification operations on a job.
By default, nobody else besides job-owner, the user who started the cluster,
members of supergroup and queue administrators can perform modification
operations on a job.
</description>
</property>
<property>
<name>mapreduce.job.acl-view-job</name>
<value> </value>
<description> Job specific access-control list for 'viewing' the job. It is
only used if authorization is enabled in Map/Reduce by setting the
configuration property mapreduce.cluster.acls.enabled to true.
This specifies the list of users and/or groups who can view private details
about the job. For specifying a list of users and groups the
format to use is "user1,user2 group1,group". If set to '*', it allows all
users/groups to modify this job. If set to ' '(i.e. space), it allows
none. This configuration is used to guard some of the job-views and at
present only protects APIs that can return possibly sensitive information
of the job-owner like
o job-level counters
o task-level counters
o tasks' diagnostic information
o task-logs displayed on the HistoryServer's web-UI and
o job.xml showed by the HistoryServer's web-UI
Every other piece of information of jobs is still accessible by any other
user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc.
Irrespective of this ACL configuration, (a) job-owner, (b) the user who
started the cluster, (c) members of an admin configured supergroup
configured via mapreduce.cluster.permissions.supergroup and (d) queue
administrators of the queue to which this job was submitted to configured
via acl-administer-jobs for the specific queue in mapred-queues.xml can
do all the view operations on a job.
By default, nobody else besides job-owner, the user who started the
cluster, memebers of supergroup and queue administrators can perform
view operations on a job.
</description>
</property>
<property>
<name>mapreduce.job.finish-when-all-reducers-done</name>
<value>true</value>
<description>Specifies whether the job should complete once all reducers
have finished, regardless of whether there are still running mappers.
</description>
</property>
<property>
<name>mapreduce.job.token.tracking.ids.enabled</name>
<value>false</value>
<description>Whether to write tracking ids of tokens to
job-conf. When true, the configuration property
"mapreduce.job.token.tracking.ids" is set to the token-tracking-ids of
the job</description>
</property>
<property>
<name>mapreduce.job.token.tracking.ids</name>
<value></value>
<description>When mapreduce.job.token.tracking.ids.enabled is
set to true, this is set by the framework to the
token-tracking-ids used by the job.</description>
</property>
<property>
<name>mapreduce.task.merge.progress.records</name>
<value>10000</value>
<description> The number of records to process during merge before
sending a progress notification to the MR ApplicationMaster.
</description>
</property>
<property>
<name>mapreduce.task.combine.progress.records</name>
<value>10000</value>
<description> The number of records to process during combine output collection
before sending a progress notification.
</description>
</property>
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
<value>0.05</value>
<description>Fraction of the number of maps in the job which should be
complete before reduces are scheduled for the job.
</description>
</property>
<property>
<name>mapreduce.job.complete.cancel.delegation.tokens</name>
<value>true</value>
<description> if false - do not unregister/cancel delegation tokens from
renewal, because same tokens may be used by spawned jobs
</description>
</property>
<property>
<name>mapreduce.shuffle.port</name>
<value>13562</value>
<description>Default port that the ShuffleHandler will run on. ShuffleHandler
is a service run at the NodeManager to facilitate transfers of intermediate
Map outputs to requesting Reducers.
</description>
</property>
<property>
<name>mapreduce.job.reduce.shuffle.consumer.plugin.class</name>
<value>org.apache.hadoop.mapreduce.task.reduce.Shuffle</value>
<description>
Name of the class whose instance will be used
to send shuffle requests by reducetasks of this job.
The class must be an instance of org.apache.hadoop.mapred.ShuffleConsumerPlugin.
</description>
</property>
<!-- MR YARN Application properties -->
<property>
<name>mapreduce.job.node-label-expression</name>
<description>All the containers of the Map Reduce job will be run with this
node label expression. If the node-label-expression for job is not set, then
it will use queue's default-node-label-expression for all job's containers.
</description>
</property>
<property>
<name>mapreduce.job.am.node-label-expression</name>
<description>This is node-label configuration for Map Reduce Application Master
container. If not configured it will make use of
mapreduce.job.node-label-expression and if job's node-label expression is not
configured then it will use queue's default-node-label-expression.
</description>
</property>
<property>
<name>mapreduce.map.node-label-expression</name>
<description>This is node-label configuration for Map task containers. If not
configured it will use mapreduce.job.node-label-expression and if job's
node-label expression is not configured then it will use queue's
default-node-label-expression.
</description>
</property>
<property>
<name>mapreduce.reduce.node-label-expression</name>
<description>This is node-label configuration for Reduce task containers. If
not configured it will use mapreduce.job.node-label-expression and if job's
node-label expression is not configured then it will use queue's
default-node-label-expression.
</description>
</property>
<property>
<name>mapreduce.job.counters.limit</name>
<value>120</value>
<description>Limit on the number of user counters allowed per job.
</description>
</property>
<property>
<name>mapreduce.framework.name</name>
<value>local</value>
<description>The runtime framework for executing MapReduce jobs.
Can be one of local, classic or yarn.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/tmp/hadoop-yarn/staging</value>
<description>The staging dir used while submitting jobs.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled</name>
<value>false</value>
<description>Whether Erasure Coding should be enabled for
files that are copied to the MR staging area. This is a job-level
setting.
</description>
</property>
<property>
<name>mapreduce.am.max-attempts</name>
<value>2</value>
<description>The maximum number of application attempts. It is a
application-specific setting. It should not be larger than the global number
set by resourcemanager. Otherwise, it will be override. The default number is
set to 2, to allow at least one retry for AM.</description>
</property>
<!-- Job Notification Configuration -->
<property>
<name>mapreduce.job.end-notification.url</name>
<!--<value>http://localhost:8080/jobstatus.phpjobId=$jobId&jobStatus=$jobStatus</value>-->
<description>Indicates url which will be called on completion of job to inform
end status of job.
User can give at most 2 variables with URI : $jobId and $jobStatus.
If they are present in URI, then they will be replaced by their
respective values.
</description>
</property>
<property>
<name>mapreduce.job.end-notification.retry.attempts</name>
<value>0</value>
<description>The number of times the submitter of the job wants to retry job
end notification if it fails. This is capped by
mapreduce.job.end-notification.max.attempts</description>
</property>
<property>
<name>mapreduce.job.end-notification.retry.interval</name>
<value>1000</value>
<description>The number of milliseconds the submitter of the job wants to
wait before job end notification is retried if it fails. This is capped by
mapreduce.job.end-notification.max.retry.interval</description>
</property>
<property>
<name>mapreduce.job.end-notification.max.attempts</name>
<value>5</value>
<final>true</final>
<description>The maximum number of times a URL will be read for providing job
end notification. Cluster administrators can set this to limit how long
after end of a job, the Application Master waits before exiting. Must be
marked as final to prevent users from overriding this.
</description>
</property>
<property>
<name>mapreduce.job.log4j-properties-file</name>
<value></value>
<description>Used to override the default settings of log4j in container-log4j.properties
for NodeManager. Like container-log4j.properties, it requires certain
framework appenders properly defined in this overriden file. The file on the
path will be added to distributed cache and classpath. If no-scheme is given
in the path, it defaults to point to a log4j file on the local FS.
</description>
</property>
<property>
<name>mapreduce.job.end-notification.max.retry.interval</name>
<value>5000</value>
<final>true</final>
<description>The maximum amount of time (in milliseconds) to wait before
retrying job end notification. Cluster administrators can set this to
limit how long the Application Master waits before exiting. Must be marked
as final to prevent users from overriding this.</description>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value></value>
<description>User added environment variables for the MR App Master
processes. Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit tasktracker's B env variable.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.admin.user.env</name>
<value></value>
<description> Environment variables for the MR App Master
processes for admin purposes. These values are set first and can be
overridden by the user env (yarn.app.mapreduce.am.env) Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit app master's B env variable.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
<value>-Xmx1024m</value>
<description>Java opts for the MR App Master processes.
The following symbol, if present, will be interpolated: @taskid@ is replaced
by current TaskID. Any other occurrences of '@' will go unchanged.
For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
Usage of -Djava.library.path can cause programs to no longer function if
hadoop native libraries are used. These values should instead be set as part
of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
mapreduce.reduce.env config settings.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.admin-command-opts</name>
<value></value>
<description>Java opts for the MR App Master processes for admin purposes.
It will appears before the opts set by yarn.app.mapreduce.am.command-opts and
thus its options can be overridden user.
Usage of -Djava.library.path can cause programs to no longer function if
hadoop native libraries are used. These values should instead be set as part
of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
mapreduce.reduce.env config settings.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.job.task.listener.thread-count</name>
<value>30</value>
<description>The number of threads used to handle RPC calls in the
MR AppMaster from remote tasks</description>
</property>
<property>
<name>yarn.app.mapreduce.am.job.client.port-range</name>
<value></value>
<description>Range of ports that the MapReduce AM can use when binding.
Leave blank if you want all possible ports.
For example 50000-50050,50100-50200</description>
</property>
<property>
<name>yarn.app.mapreduce.am.webapp.port-range</name>
<value></value>
<description>Range of ports that the MapReduce AM can use for its webapp when binding.
Leave blank if you want all possible ports.
For example 50000-50050,50100-50200</description>
</property>
<property>
<name>yarn.app.mapreduce.am.job.committer.cancel-timeout</name>
<value>60000</value>
<description>The amount of time in milliseconds to wait for the output
committer to cancel an operation if the job is killed</description>
</property>
<property>
<name>yarn.app.mapreduce.am.job.committer.commit-window</name>
<value>10000</value>
<description>Defines a time window in milliseconds for output commit
operations. If contact with the RM has occurred within this window then
commits are allowed, otherwise the AM will not allow output commits until
contact with the RM has been re-established.</description>
</property>
<property>
<name>mapreduce.fileoutputcommitter.algorithm.version</name>
<value>2</value>
<description>The file output committer algorithm version
valid algorithm version number: 1 or 2
default to 2, which is the original algorithm
In algorithm version 1,
1. commitTask will rename directory
$joboutput/_temporary/$appAttemptID/_temporary/$taskAttemptID/
to
$joboutput/_temporary/$appAttemptID/$taskID/
2. recoverTask will also do a rename
$joboutput/_temporary/$appAttemptID/$taskID/
to
$joboutput/_temporary/($appAttemptID + 1)/$taskID/
3. commitJob will merge every task output file in
$joboutput/_temporary/$appAttemptID/$taskID/
to
$joboutput/, then it will delete $joboutput/_temporary/
and write $joboutput/_SUCCESS
It has a performance regression, which is discussed in MAPREDUCE-4815.
If a job generates many files to commit then the commitJob
method call at the end of the job can take minutes.
the commit is single-threaded and waits until all
tasks have completed before commencing.
algorithm version 2 will change the behavior of commitTask,
recoverTask, and commitJob.
1. commitTask will rename all files in
$joboutput/_temporary/$appAttemptID/_temporary/$taskAttemptID/
to $joboutput/
2. recoverTask actually doesn't require to do anything, but for
upgrade from version 1 to version 2 case, it will check if there
are any files in
$joboutput/_temporary/($appAttemptID - 1)/$taskID/
and rename them to $joboutput/
3. commitJob can simply delete $joboutput/_temporary and write
$joboutput/_SUCCESS
This algorithm will reduce the output commit time for
large jobs by having the tasks commit directly to the final
output directory as they were completing and commitJob had
very little to do.
</description>
</property>
<property>
<name>mapreduce.fileoutputcommitter.task.cleanup.enabled</name>
<value>false</value>
<description>Whether tasks should delete their task temporary directories. This is purely an
optimization for filesystems without O(1) recursive delete, as commitJob will recursively delete
the entire job temporary directory. HDFS has O(1) recursive delete, so this parameter is left
false by default. Users of object stores, for example, may want to set this to true.
Note: this is only used if mapreduce.fileoutputcommitter.algorithm.version=2</description>
</property>
<property>
<name>yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms</name>
<value>1000</value>
<description>The interval in ms at which the MR AppMaster should send
heartbeats to the ResourceManager</description>
</property>
<property>
<name>yarn.app.mapreduce.client-am.ipc.max-retries</name>
<value>3</value>
<description>The number of client retries to the AM - before reconnecting
to the RM to fetch Application Status.</description>
</property>
<property>
<name>yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts</name>
<value>3</value>
<description>The number of client retries on socket timeouts to the AM - before
reconnecting to the RM to fetch Application Status.</description>
</property>
<property>
<name>yarn.app.mapreduce.client.max-retries</name>
<value>3</value>
<description>The number of client retries to the RM/HS before
throwing exception. This is a layer above the ipc.</description>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>1536</value>
<description>The amount of memory the MR AppMaster needs.</description>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
<value>1</value>
<description>
The number of virtual CPU cores the MR AppMaster needs.
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.hard-kill-timeout-ms</name>
<value>10000</value>
<description>
Number of milliseconds to wait before the job client kills the application.
</description>
</property>
<property>
<name>yarn.app.mapreduce.client.job.max-retries</name>
<value>3</value>
<description>The number of retries the client will make for getJob and
dependent calls.
This is needed for non-HDFS DFS where additional, high level
retries are required to avoid spurious failures during the getJob call.
30 is a good value for WASB</description>
</property>
<property>
<name>yarn.app.mapreduce.client.job.retry-interval</name>
<value>2000</value>
<description>The delay between getJob retries in ms for retries configured
with yarn.app.mapreduce.client.job.max-retries.</description>
</property>
<property>
<description>CLASSPATH for MR applications. A comma-separated list
of CLASSPATH entries. If mapreduce.application.framework is set then this
must specify the appropriate classpath for that archive, and the name of
the archive must be present in the classpath.
If mapreduce.app-submission.cross-platform is false, platform-specific
environment vairable expansion syntax would be used to construct the default
CLASSPATH entries.
For Linux:
$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,
$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*.
For Windows:
%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,
%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*.
If mapreduce.app-submission.cross-platform is true, platform-agnostic default
CLASSPATH for MR applications would be used:
{{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/*,
{{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/lib/*
Parameter expansion marker will be replaced by NodeManager on container
launch based on the underlying OS accordingly.
</description>
<name>mapreduce.application.classpath</name>
<value></value>
</property>
<property>
<description>If enabled, user can submit an application cross-platform
i.e. submit an application from a Windows client to a Linux/Unix server or
vice versa.
</description>
<name>mapreduce.app-submission.cross-platform</name>
<value>false</value>
</property>
<property>
<description>Path to the MapReduce framework archive. If set, the framework
archive will automatically be distributed along with the job, and this
path would normally reside in a public location in an HDFS filesystem. As
with distributed cache files, this can be a URL with a fragment specifying
the alias to use for the archive name. For example,
hdfs:/mapred/framework/hadoop-mapreduce-2.1.1.tar.gz#mrframework would
alias the localized archive as "mrframework".
Note that mapreduce.application.classpath must include the appropriate
classpath for the specified framework. The base name of the archive, or
alias of the archive if an alias is used, must appear in the specified
classpath.
</description>
<name>mapreduce.application.framework.path</name>
<value></value>
</property>
<property>
<name>mapreduce.job.classloader</name>
<value>false</value>
<description>Whether to use a separate (isolated) classloader for
user classes in the task JVM.</description>
</property>
<property>
<name>mapreduce.job.classloader.system.classes</name>
<value></value>
<description>Used to override the default definition of the system classes for
the job classloader. The system classes are a comma-separated list of
patterns that indicate whether to load a class from the system classpath,
instead from the user-supplied JARs, when mapreduce.job.classloader is
enabled.
A positive pattern is defined as:
1. A single class name 'C' that matches 'C' and transitively all nested
classes 'C$*' defined in C;
2. A package name ending with a '.' (e.g., "com.example.") that matches
all classes from that package.
A negative pattern is defined by a '-' in front of a positive pattern
(e.g., "-com.example.").
A class is considered a system class if and only if it matches one of the
positive patterns and none of the negative ones. More formally:
A class is a member of the inclusion set I if it matches one of the positive
patterns. A class is a member of the exclusion set E if it matches one of
the negative patterns. The set of system classes S = I \ E.
</description>
</property>
<property>
<name>mapreduce.jvm.system-properties-to-log</name>
<value>os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name</value>
<description>Comma-delimited list of system properties to log on mapreduce JVM start</description>
</property>
<!-- jobhistory properties -->
<property>
<name>mapreduce.jobhistory.address</name>
<value>0.0.0.0:10020</value>
<description>MapReduce JobHistory Server IPC host:port</description>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>0.0.0.0:19888</value>
<description>MapReduce JobHistory Server Web UI host:port</description>
</property>
<property>
<name>mapreduce.jobhistory.webapp.https.address</name>
<value>0.0.0.0:19890</value>
<description>
The https address the MapReduce JobHistory Server WebApp is on.
</description>
</property>
<property>
<name>mapreduce.jobhistory.keytab</name>
<description>
Location of the kerberos keytab file for the MapReduce
JobHistory Server.
</description>
<value>/etc/security/keytab/jhs.service.keytab</value>
</property>
<property>
<name>mapreduce.jobhistory.principal</name>
<description>
Kerberos principal name for the MapReduce JobHistory Server.
</description>
<value>jhs/_HOST@REALM.TLD</value>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-done-dir</name>
<value>${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate</value>
<description></description>
</property>
<property>
<name>mapreduce.jobhistory.intermediate-user-done-dir.permissions</name>
<value>770</value>
<description>The permissions of the user directories in
${mapreduce.jobhistory.intermediate-done-dir}. The user and the group
permission must be 7, this is enforced.
</description>
</property>
<property>
<name>mapreduce.jobhistory.always-scan-user-dir</name>
<value>false</value>
<description>Some Cloud FileSystems do not currently update the
modification time of directories. To support these filesystems, this
configuration value should be set to 'true'.
</description>
</property>
<property>
<name>mapreduce.jobhistory.done-dir</name>
<value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>
<description></description>
</property>
<property>
<name>mapreduce.jobhistory.cleaner.enable</name>
<value>true</value>
<description></description>
</property>
<property>
<name>mapreduce.jobhistory.cleaner.interval-ms</name>
<value>86400000</value>
<description> How often the job history cleaner checks for files to delete,
in milliseconds. Defaults to 86400000 (one day). Files are only deleted if
they are older than mapreduce.jobhistory.max-age-ms.
</description>
</property>
<property>
<name>mapreduce.jobhistory.max-age-ms</name>
<value>604800000</value>
<description> Job history files older than this many milliseconds will
be deleted when the history cleaner runs. Defaults to 604800000 (1 week).
</description>
</property>
<property>
<name>mapreduce.jobhistory.client.thread-count</name>
<value>10</value>
<description>The number of threads to handle client API requests</description>
</property>
<property>
<name>mapreduce.jobhistory.datestring.cache.size</name>
<value>200000</value>
<description>Size of the date string cache. Effects the number of directories
which will be scanned to find a job.</description>
</property>
<property>
<name>mapreduce.jobhistory.joblist.cache.size</name>
<value>20000</value>
<description>Size of the job list cache</description>
</property>
<property>
<name>mapreduce.jobhistory.loadedjobs.cache.size</name>
<value>5</value>
<description>Size of the loaded job cache. This property is ignored if
the property mapreduce.jobhistory.loadedtasks.cache.size is set to a
positive value.
</description>
</property>
<property>
<name>mapreduce.jobhistory.loadedtasks.cache.size</name>
<value></value>
<description>Change the job history cache limit to be set in terms
of total task count. If the total number of tasks loaded exceeds
this value, then the job cache will be shrunk down until it is
under this limit (minimum 1 job in cache). If this value is empty
or nonpositive then the cache reverts to using the property
mapreduce.jobhistory.loadedjobs.cache.size as a job cache size.
Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size
property:
1) For every 100k of cache size, set the heap size of the Job History
Server to 1.2GB. For example,
mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB.
2) Make sure that the cache size is larger than the number of tasks
required for the largest job run on the cluster. It might be a good
idea to set the value slightly higher (say, 20%) in order to allow
for job size growth.
</description>
</property>
<property>
<name>mapreduce.jobhistory.move.interval-ms</name>
<value>180000</value>
<description>Scan for history files to more from intermediate done dir to done
dir at this frequency.
</description>
</property>
<property>
<name>mapreduce.jobhistory.move.thread-count</name>
<value>3</value>
<description>The number of threads used to move files.</description>
</property>
<property>
<name>mapreduce.jobhistory.store.class</name>
<value></value>
<description>The HistoryStorage class to use to cache history data.</description>
</property>
<property>
<name>mapreduce.jobhistory.minicluster.fixed.ports</name>
<value>false</value>
<description>Whether to use fixed ports with the minicluster</description>
</property>
<property>
<name>mapreduce.jobhistory.admin.address</name>
<value>0.0.0.0:10033</value>
<description>The address of the History server admin interface.</description>
</property>
<property>
<name>mapreduce.jobhistory.admin.acl</name>
<value>*</value>
<description>ACL of who can be admin of the History server.</description>
</property>
<property>
<name>mapreduce.jobhistory.recovery.enable</name>
<value>false</value>
<description>Enable the history server to store server state and recover
server state upon startup. If enabled then
mapreduce.jobhistory.recovery.store.class must be specified.</description>
</property>
<property>
<name>mapreduce.jobhistory.recovery.store.class</name>
<value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService</value>
<description>The HistoryServerStateStoreService class to store history server
state for recovery.</description>
</property>
<property>
<name>mapreduce.jobhistory.recovery.store.fs.uri</name>
<value>${hadoop.tmp.dir}/mapred/history/recoverystore</value>
<!--value>hdfs://localhost:9000/mapred/history/recoverystore</value-->
<description>The URI where history server state will be stored if
HistoryServerFileSystemStateStoreService is configured as the recovery
storage class.</description>
</property>
<property>
<name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
<value>${hadoop.tmp.dir}/mapred/history/recoverystore</value>
<description>The URI where history server state will be stored if
HistoryServerLeveldbSystemStateStoreService is configured as the recovery
storage class.</description>
</property>
<property>
<name>mapreduce.jobhistory.http.policy</name>
<value>HTTP_ONLY</value>
<description>
This configures the HTTP endpoint for JobHistoryServer web UI.
The following values are supported:
- HTTP_ONLY : Service is provided only on http
- HTTPS_ONLY : Service is provided only on https
</description>
</property>
<property>
<name>mapreduce.jobhistory.jobname.limit</name>
<value>50</value>
<description>
Number of characters allowed for job name in Job History Server web page.
</description>
</property>
<property>
<description>
File format the AM will use when generating the .jhist file. Valid
values are "json" for text output and "binary" for faster parsing.
</description>
<name>mapreduce.jobhistory.jhist.format</name>
<value>binary</value>
</property>
<property>
<name>mapreduce.job.heap.memory-mb.ratio</name>
<value>0.8</value>
<description>The ratio of heap-size to container-size. If no -Xmx is
specified, it is calculated as
(mapreduce.{map|reduce}.memory.mb * mapreduce.heap.memory-mb.ratio).
If -Xmx is specified but not mapreduce.{map|reduce}.memory.mb, it is
calculated as (heapSize / mapreduce.heap.memory-mb.ratio).
</description>
</property>
<property>
<name>yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size</name>
<value>10</value>
<description>The initial size of thread pool to launch containers in the
app master.
</description>
</property>
<property>
<name>mapreduce.task.exit.timeout</name>
<value>60000</value>
<description>The number of milliseconds before a task will be
terminated if it stays in finishing state for too long.
After a task attempt completes from TaskUmbilicalProtocol's point of view,
it will be transitioned to finishing state. That will give a chance for the
task to exit by itself.
</description>
</property>
<property>
<name>mapreduce.task.exit.timeout.check-interval-ms</name>
<value>20000</value>
<description>The interval in milliseconds between which the MR framework
checks if task attempts stay in finishing state for too long.
</description>
</property>
<property>
<name>mapreduce.job.encrypted-intermediate-data</name>
<value>false</value>
<description>Encrypt intermediate MapReduce spill files or not
default is false</description>
</property>
<property>
<name>mapreduce.job.encrypted-intermediate-data-key-size-bits</name>
<value>128</value>
<description>Mapreduce encrypt data key size default is 128</description>
</property>
<property>
<name>mapreduce.job.encrypted-intermediate-data.buffer.kb</name>
<value>128</value>
<description>Buffer size for intermediate encrypt data in kb
default is 128</description>
</property>
<property>
<name>mapreduce.task.local-fs.write-limit.bytes</name>
<value>-1</value>
<description>Limit on the byte written to the local file system by each task.
This limit only applies to writes that go through the Hadoop filesystem APIs
within the task process (i.e.: writes that will update the local filesystem's
BYTES_WRITTEN counter). It does not cover other writes such as logging,
sideband writes from subprocesses (e.g.: streaming jobs), etc.
Negative values disable the limit.
default is -1</description>
</property>
<property>
<description>
Enable the CSRF filter for the job history web app
</description>
<name>mapreduce.jobhistory.webapp.rest-csrf.enabled</name>
<value>false</value>
</property>
<property>
<description>
Optional parameter that indicates the custom header name to use for CSRF
protection.
</description>
<name>mapreduce.jobhistory.webapp.rest-csrf.custom-header</name>
<value>X-XSRF-Header</value>
</property>
<property>
<description>
Optional parameter that indicates the list of HTTP methods that do not
require CSRF protection
</description>
<name>mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD</value>
</property>
<property>
<name>mapreduce.job.cache.limit.max-resources</name>
<value>0</value>
<description>The maximum number of resources a map reduce job is allowed to
submit for localization via files, libjars, archives, and jobjar command
line arguments and through the distributed cache. If set to 0 the limit is
ignored.
</description>
</property>
<property>
<name>mapreduce.job.cache.limit.max-resources-mb</name>
<value>0</value>
<description>The maximum size (in MB) a map reduce job is allowed to submit
for localization via files, libjars, archives, and jobjar command line
arguments and through the distributed cache. If set to 0 the limit is
ignored.
</description>
</property>
<property>
<name>mapreduce.job.cache.limit.max-single-resource-mb</name>
<value>0</value>
<description>The maximum size (in MB) of a single resource a map reduce job
is allow to submit for localization via files, libjars, archives, and
jobjar command line arguments and through the distributed cache. If set to
0 the limit is ignored.
</description>
</property>
<property>
<description>
Value of the xframe-options
</description>
<name>mapreduce.jobhistory.webapp.xfs-filter.xframe-options</name>
<value>SAMEORIGIN</value>
</property>
<property>
<description>
The maximum number of tasks that a job can have so that the Job History
Server will fully parse its associated job history file and load it into
memory. A value of -1 (default) will allow all jobs to be loaded.
</description>
<name>mapreduce.jobhistory.loadedjob.tasks.max</name>
<value>-1</value>
</property>
<property>
<description>
The list of job configuration properties whose value will be redacted.
</description>
<name>mapreduce.job.redacted-properties</name>
<value></value>
</property>
<property>
<description>
This configuration is a regex expression. The list of configurations that
match the regex expression will be sent to RM. RM will use these
configurations for renewing tokens.
This configuration is added for below scenario: User needs to run distcp
jobs across two clusters, but the RM does not have necessary hdfs
configurations to connect to the remote hdfs cluster. Hence, user relies on
this config to send the configurations to RM and RM uses these
configurations to renew tokens.
For example the following regex expression indicates the minimum required
configs for RM to connect to a remote hdfs cluster:
dfs.nameservices|^dfs.namenode.rpc-address.*$|^dfs.ha.namenodes.*$|^dfs.client.failover.proxy.provider.*$|dfs.namenode.kerberos.principal
</description>
<name>mapreduce.job.send-token-conf</name>
<value></value>
</property>
<property>
<description>
The name of an output committer factory for MRv2 FileOutputFormat to use
for committing work. If set, overrides any per-filesystem committer
defined for the destination filesystem.
</description>
<name>mapreduce.outputcommitter.factory.class</name>
<value></value>
</property>
<property>
<name>mapreduce.outputcommitter.factory.scheme.s3a</name>
<value>org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory</value>
<description>
The committer factory to use when writing data to S3A filesystems.
If mapreduce.outputcommitter.factory.class is set, it will
override this property.
</description>
</property>
</configuration>
4.yarn-default.xml
<xml version="1.0">
<xml-stylesheet type="text/xsl" href="configuration.xsl">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Do not modify this file directly. Instead, copy entries that you -->
<!-- wish to modify from this file into yarn-site.xml and change them -->
<!-- there. If yarn-site.xml does not already exist, create it. -->
<configuration>
<!-- IPC Configuration -->
<property>
<description>Factory to create client IPC classes.</description>
<name>yarn.ipc.client.factory.class</name>
</property>
<property>
<description>Factory to create server IPC classes.</description>
<name>yarn.ipc.server.factory.class</name>
</property>
<property>
<description>Factory to create serializeable records.</description>
<name>yarn.ipc.record.factory.class</name>
</property>
<property>
<description>RPC class implementation</description>
<name>yarn.ipc.rpc.class</name>
<value>org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC</value>
</property>
<!-- Resource Manager Configuration -->
<property>
<description>The hostname of the RM.</description>
<name>yarn.resourcemanager.hostname</name>
<value>0.0.0.0</value>
</property>
<property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
<value>${yarn.resourcemanager.hostname}:8032</value>
</property>
<property>
<description>
The actual address the server will bind to. If this optional address is
set, the RPC and webapp servers will bind to this address and the port specified in
yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively. This
is most useful for making RM listen to all interfaces by setting to 0.0.0.0.
</description>
<name>yarn.resourcemanager.bind-host</name>
<value></value>
</property>
<property>
<description>
If set to true, then ALL container updates will be automatically sent to
the NM in the next heartbeat</description>
<name>yarn.resourcemanager.auto-update.containers</name>
<value>false</value>
</property>
<property>
<description>The number of threads used to handle applications manager requests.</description>
<name>yarn.resourcemanager.client.thread-count</name>
<value>50</value>
</property>
<property>
<description>Number of threads used to launch/cleanup AM.</description>
<name>yarn.resourcemanager.amlauncher.thread-count</name>
<value>50</value>
</property>
<property>
<description>Retry times to connect with NM.</description>
<name>yarn.resourcemanager.nodemanager-connect-retries</name>
<value>10</value>
</property>
<property>
<description>Timeout in milliseconds when YARN dispatcher tries to drain the
events. Typically, this happens when service is stopping. e.g. RM drains
the ATS events dispatcher when stopping.
</description>
<name>yarn.dispatcher.drain-events.timeout</name>
<value>300000</value>
</property>
<property>
<description>The expiry interval for application master reporting.</description>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
</property>
<property>
<description>The Kerberos principal for the resource manager.</description>
<name>yarn.resourcemanager.principal</name>
</property>
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
<value>${yarn.resourcemanager.hostname}:8030</value>
</property>
<property>
<description>Number of threads to handle scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.client.thread-count</name>
<value>50</value>
</property>
<property>
<description>
Specify which handler will be used to process PlacementConstraints.
Acceptable values are: `placement-processor`, `scheduler` and `disabled`.
For a detailed explanation of these values, please refer to documentation.
</description>
<name>yarn.resourcemanager.placement-constraints.handler</name>
<value>disabled</value>
</property>
<property>
<description>Number of times to retry placing of rejected SchedulingRequests</description>
<name>yarn.resourcemanager.placement-constraints.retry-attempts</name>
<value>3</value>
</property>
<property>
<description>Constraint Placement Algorithm to be used.</description>
<name>yarn.resourcemanager.placement-constraints.algorithm.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm</value>
</property>
<property>
<description>Placement Algorithm Requests Iterator to be used.</description>
<name>yarn.resourcemanager.placement-constraints.algorithm.iterator</name>
<value>SERIAL</value>
</property>
<property>
<description>Threadpool size for the Algorithm used for placement constraint processing.</description>
<name>yarn.resourcemanager.placement-constraints.algorithm.pool-size</name>
<value>1</value>
</property>
<property>
<description>Threadpool size for the Scheduler invocation phase of placement constraint processing.</description>
<name>yarn.resourcemanager.placement-constraints.scheduler.pool-size</name>
<value>1</value>
</property>
<property>
<description>
Comma separated class names of ApplicationMasterServiceProcessor
implementations. The processors will be applied in the order
they are specified.
</description>
<name>yarn.resourcemanager.application-master-service.processors</name>
<value></value>
</property>
<property>
<description>
This configures the HTTP endpoint for YARN Daemons.The following
values are supported:
- HTTP_ONLY : Service is provided only on http
- HTTPS_ONLY : Service is provided only on https
</description>
<name>yarn.http.policy</name>
<value>HTTP_ONLY</value>
</property>
<property>
<description>
The http address of the RM web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name>yarn.resourcemanager.webapp.address</name>
<value>${yarn.resourcemanager.hostname}:8088</value>
</property>
<property>
<description>
The https address of the RM web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name>yarn.resourcemanager.webapp.https.address</name>
<value>${yarn.resourcemanager.hostname}:8090</value>
</property>
<property>
<description>
The Kerberos keytab file to be used for spnego filter for the RM web
interface.
</description>
<name>yarn.resourcemanager.webapp.spnego-keytab-file</name>
<value></value>
</property>
<property>
<description>
The Kerberos principal to be used for spnego filter for the RM web
interface.
</description>
<name>yarn.resourcemanager.webapp.spnego-principal</name>
<value></value>
</property>
<property>
<description>
Add button to kill application in the RM Application view.
</description>
<name>yarn.resourcemanager.webapp.ui-actions.enabled</name>
<value>true</value>
</property>
<property>
<description>To enable RM web ui2 application.</description>
<name>yarn.webapp.ui2.enable</name>
<value>false</value>
</property>
<property>
<description>
Explicitly provide WAR file path for ui2 if needed.
</description>
<name>yarn.webapp.ui2.war-file-path</name>
<value></value>
</property>
<property>
<description>
Enable services rest api on ResourceManager.
</description>
<name>yarn.webapp.api-service.enable</name>
<value>false</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>
</property>
<property>
<description>Are acls enabled.</description>
<name>yarn.acl.enable</name>
<value>false</value>
</property>
<property>
<description>Are reservation acls enabled.</description>
<name>yarn.acl.reservation-enable</name>
<value>false</value>
</property>
<property>
<description>ACL of who can be admin of the YARN cluster.</description>
<name>yarn.admin.acl</name>
<value>*</value>
</property>
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
<value>${yarn.resourcemanager.hostname}:8033</value>
</property>
<property>
<description>Number of threads used to handle RM admin interface.</description>
<name>yarn.resourcemanager.admin.client.thread-count</name>
<value>1</value>
</property>
<property>
<description>Maximum time to wait to establish connection to
ResourceManager.</description>
<name>yarn.resourcemanager.connect.max-wait.ms</name>
<value>900000</value>
</property>
<property>
<description>How often to try connecting to the
ResourceManager.</description>
<name>yarn.resourcemanager.connect.retry-interval.ms</name>
<value>30000</value>
</property>
<property>
<description>The maximum number of application attempts. It's a global
setting for all application masters. Each application master can specify
its individual maximum number of application attempts via the API, but the
individual number cannot be more than the global upper bound. If it is,
the resourcemanager will override it. The default number is set to 2, to
allow at least one retry for AM.</description>
<name>yarn.resourcemanager.am.max-attempts</name>
<value>2</value>
</property>
<property>
<description>How often to check that containers are still alive. </description>
<name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
<value>600000</value>
</property>
<property>
<description>The keytab for the resource manager.</description>
<name>yarn.resourcemanager.keytab</name>
<value>/etc/krb5.keytab</value>
</property>
<property>
<description>Flag to enable override of the default kerberos authentication
filter with the RM authentication filter to allow authentication using
delegation tokens(fallback to kerberos if the tokens are missing). Only
applicable when the http authentication type is kerberos.</description>
<name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
<value>true</value>
</property>
<property>
<description>Flag to enable cross-origin (CORS) support in the RM. This flag
requires the CORS filter initializer to be added to the filter initializers
list in core-site.xml.</description>
<name>yarn.resourcemanager.webapp.cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>How long to wait until a node manager is considered dead.</description>
<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
</property>
<property>
<description>Path to file with nodes to include.</description>
<name>yarn.resourcemanager.nodes.include-path</name>
<value></value>
</property>
<property>
<description>Path to file with nodes to exclude.</description>
<name>yarn.resourcemanager.nodes.exclude-path</name>
<value></value>
</property>
<property>
<description>The expiry interval for node IP caching. -1 disables the caching</description>
<name>yarn.resourcemanager.node-ip-cache.expiry-interval-secs</name>
<value>-1</value>
</property>
<property>
<description>Number of threads to handle resource tracker calls.</description>
<name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
<value>50</value>
</property>
<property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>
<property>
<description>The minimum allocation for every container request at the RM
in MBs. Memory requests lower than this will be set to the value of this
property. Additionally, a node manager that is configured to have less memory
than this value will be shut down by the resource manager.</description>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>1024</value>
</property>
<property>
<description>The maximum allocation for every container request at the RM
in MBs. Memory requests higher than this will throw an
InvalidResourceRequestException.</description>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>8192</value>
</property>
<property>
<description>The minimum allocation for every container request at the RM
in terms of virtual CPU cores. Requests lower than this will be set to the
value of this property. Additionally, a node manager that is configured to
have fewer virtual cores than this value will be shut down by the resource
manager.</description>
<name>yarn.scheduler.minimum-allocation-vcores</name>
<value>1</value>
</property>
<property>
<description>The maximum allocation for every container request at the RM
in terms of virtual CPU cores. Requests higher than this will throw an
InvalidResourceRequestException.</description>
<name>yarn.scheduler.maximum-allocation-vcores</name>
<value>4</value>
</property>
<property>
<description>
Used by node labels. If set to true, the port should be included in the
node name. Only usable if your scheduler supports node labels.
</description>
<name>yarn.scheduler.include-port-in-node-name</name>
<value>false</value>
</property>
<property>
<description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified. </description>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>false</value>
</property>
<property>
<description>Should RM fail fast if it encounters any errors. By defalt, it
points to ${yarn.fail-fast}. Errors include:
1) exceptions when state-store write/read operations fails.
</description>
<name>yarn.resourcemanager.fail-fast</name>
<value>${yarn.fail-fast}</value>
</property>
<property>
<description>Should YARN fail fast if it encounters any errors.
This is a global config for all other components including RM,NM etc.
If no value is set for component-specific config (e.g yarn.resourcemanager.fail-fast),
this value will be the default.
</description>
<name>yarn.fail-fast</name>
<value>false</value>
</property>
<property>
<description>Enable RM work preserving recovery. This configuration is private
to YARN for experimenting the feature.
</description>
<name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
<value>true</value>
</property>
<property>
<description>Set the amount of time RM waits before allocating new
containers on work-preserving-recovery. Such wait period gives RM a chance
to settle down resyncing with NMs in the cluster on recovery, before assigning
new containers to applications.
</description>
<name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
<value>10000</value>
</property>
<property>
<description>The class to use as the persistent store.
If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
is used, the store is implicitly fenced; meaning a single ResourceManager
is able to use the store at any point in time. More details on this
implicit fencing, along with setting up appropriate ACLs is discussed
under yarn.resourcemanager.zk-state-store.root-node.acl.
</description>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore</value>
</property>
<property>
<description>When automatic failover is enabled, number of zookeeper
operation retry times in ActiveStandbyElector</description>
<name>yarn.resourcemanager.ha.failover-controller.active-standby-elector.zk.retries</name>
<!--<value>3</value>-->
</property>
<property>
<description>The maximum number of completed applications RM state
store keeps, less than or equals to ${yarn.resourcemanager.max-completed-applications}.
By default, it equals to ${yarn.resourcemanager.max-completed-applications}.
This ensures that the applications kept in the state store are consistent with
the applications remembered in RM memory.
Any values larger than ${yarn.resourcemanager.max-completed-applications} will
be reset to ${yarn.resourcemanager.max-completed-applications}.
Note that this value impacts the RM recovery performance. Typically,
a smaller value indicates better performance on RM recovery.
</description>
<name>yarn.resourcemanager.state-store.max-completed-applications</name>
<value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
<property>
<description>Full path of the ZooKeeper znode where RM state will be
stored. This must be supplied when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
as the value for yarn.resourcemanager.store.class</description>
<name>yarn.resourcemanager.zk-state-store.parent-path</name>
<value>/rmstore</value>
</property>
<property>
<description>
ACLs to be used for the root znode when using ZKRMStateStore in an HA
scenario for fencing.
ZKRMStateStore supports implicit fencing to allow a single
ResourceManager write-access to the store. For fencing, the
ResourceManagers in the cluster share read-write-admin privileges on the
root node, but the Active ResourceManager claims exclusive create-delete
permissions.
By default, when this property is not set, we use the ACLs from
yarn.resourcemanager.zk-acl for shared admin access and
rm-address:random-number for username-based exclusive create-delete
access.
This property allows users to set ACLs of their choice instead of using
the default mechanism. For fencing to work, the ACLs should be
carefully set differently on each ResourceManger such that all the
ResourceManagers have shared admin access and the Active ResourceManger
takes over (exclusively) the create-delete access.
</description>
<name>yarn.resourcemanager.zk-state-store.root-node.acl</name>
</property>
<property>
<description>URI pointing to the location of the FileSystem path where
RM state will be stored. This must be supplied when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
as the value for yarn.resourcemanager.store.class</description>
<name>yarn.resourcemanager.fs.state-store.uri</name>
<value>${hadoop.tmp.dir}/yarn/system/rmstore</value>
<!--value>hdfs://localhost:9000/rmstore</value-->
</property>
<property>
<description>the number of retries to recover from IOException in
FileSystemRMStateStore.
</description>
<name>yarn.resourcemanager.fs.state-store.num-retries</name>
<value>0</value>
</property>
<property>
<description>Retry interval in milliseconds in FileSystemRMStateStore.
</description>
<name>yarn.resourcemanager.fs.state-store.retry-interval-ms</name>
<value>1000</value>
</property>
<property>
<description>Local path where the RM state will be stored when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.LeveldbRMStateStore
as the value for yarn.resourcemanager.store.class</description>
<name>yarn.resourcemanager.leveldb-state-store.path</name>
<value>${hadoop.tmp.dir}/yarn/system/rmstore</value>
</property>
<property>
<description>The time in seconds between full compactions of the leveldb
database. Setting the interval to zero disables the full compaction
cycles.</description>
<name>yarn.resourcemanager.leveldb-state-store.compaction-interval-secs</name>
<value>3600</value>
</property>
<property>
<description>Enable RM high-availability. When enabled,
(1) The RM starts in the Standby mode by default, and transitions to
the Active mode when prompted to.
(2) The nodes in the RM ensemble are listed in
yarn.resourcemanager.ha.rm-ids
(3) The id of each RM either comes from yarn.resourcemanager.ha.id
if yarn.resourcemanager.ha.id is explicitly specified or can be
figured out by matching yarn.resourcemanager.address.{id} with local address
(4) The actual physical addresses come from the configs of the pattern
- {rpc-config}.{id}</description>
<name>yarn.resourcemanager.ha.enabled</name>
<value>false</value>
</property>
<property>
<description>Enable automatic failover.
By default, it is enabled only when HA is enabled</description>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<description>Enable embedded automatic failover.
By default, it is enabled only when HA is enabled.
The embedded elector relies on the RM state store to handle fencing,
and is primarily intended to be used in conjunction with ZKRMStateStore.
</description>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
<value>true</value>
</property>
<property>
<description>The base znode path to use for storing leader information,
when using ZooKeeper based leader election.</description>
<name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>
<value>/yarn-leader-election</value>
</property>
<property>
<description>Index at which last section of application id (with each section
separated by _ in application id) will be split so that application znode
stored in zookeeper RM state store will be stored as two different znodes
(parent-child). Split is done from the end.
For instance, with no split, appid znode will be of the form
application_1352994193343_0001. If the value of this config is 1, the
appid znode will be broken into two parts application_1352994193343_000
and 1 respectively with former being the parent node.
application_1352994193343_0002 will then be stored as 2 under the parent
node application_1352994193343_000. This config can take values from 0 to 4.
0 means there will be no split. If configuration value is outside this
range, it will be treated as config value of 0(i.e. no split). A value
larger than 0 (up to 4) should be configured if you are storing a large number
of apps in ZK based RM state store and state store operations are failing due to
LenError in Zookeeper.</description>
<name>yarn.resourcemanager.zk-appid-node.split-index</name>
<value>0</value>
</property>
<property>
<description>Index at which the RM Delegation Token ids will be split so
that the delegation token znodes stored in the zookeeper RM state store
will be stored as two different znodes (parent-child). The split is done
from the end. For instance, with no split, a delegation token znode will
be of the form RMDelegationToken_123456789. If the value of this config is
1, the delegation token znode will be broken into two parts:
RMDelegationToken_12345678 and 9 respectively with former being the parent
node. This config can take values from 0 to 4. 0 means there will be no
split. If the value is outside this range, it will be treated as 0 (i.e.
no split). A value larger than 0 (up to 4) should be configured if you are
running a large number of applications, with long-lived delegation tokens
and state store operations (e.g. failover) are failing due to LenError in
Zookeeper.</description>
<name>yarn.resourcemanager.zk-delegation-token-node.split-index</name>
<value>0</value>
</property>
<property>
<description>Specifies the maximum size of the data that can be stored
in a znode. Value should be same or less than jute.maxbuffer configured
in zookeeper. Default value configured is 1MB.</description>
<name>yarn.resourcemanager.zk-max-znode-size.bytes</name>
<value>1048576</value>
</property>
<property>
<description>Name of the cluster. In a HA setting,
this is used to ensure the RM participates in leader
election for this cluster and ensures it does not affect
other clusters</description>
<name>yarn.resourcemanager.cluster-id</name>
<!--value>yarn-cluster</value-->
</property>
<property>
<description>The list of RM nodes in the cluster when HA is
enabled. See description of yarn.resourcemanager.ha
.enabled for full details on how this is used.</description>
<name>yarn.resourcemanager.ha.rm-ids</name>
<!--value>rm1,rm2</value-->
</property>
<property>
<description>The id (string) of the current RM. When HA is enabled, this
is an optional config. The id of current RM can be set by explicitly
specifying yarn.resourcemanager.ha.id or figured out by matching
yarn.resourcemanager.address.{id} with local address
See description of yarn.resourcemanager.ha.enabled
for full details on how this is used.</description>
<name>yarn.resourcemanager.ha.id</name>
<!--value>rm1</value-->
</property>
<property>
<description>When HA is enabled, the class to be used by Clients, AMs and
NMs to failover to the Active RM. It should extend
org.apache.hadoop.yarn.client.RMFailoverProxyProvider</description>
<name>yarn.client.failover-proxy-provider</name>
<value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
</property>
<property>
<description>When HA is enabled, the max number of times
FailoverProxyProvider should attempt failover. When set,
this overrides the yarn.resourcemanager.connect.max-wait.ms. When
not set, this is inferred from
yarn.resourcemanager.connect.max-wait.ms.</description>
<name>yarn.client.failover-max-attempts</name>
<!--value>15</value-->
</property>
<property>
<description>When HA is enabled, the sleep base (in milliseconds) to be
used for calculating the exponential delay between failovers. When set,
this overrides the yarn.resourcemanager.connect.* settings. When
not set, yarn.resourcemanager.connect.retry-interval.ms is used instead.
</description>
<name>yarn.client.failover-sleep-base-ms</name>
<!--value>500</value-->
</property>
<property>
<description>When HA is enabled, the maximum sleep time (in milliseconds)
between failovers. When set, this overrides the
yarn.resourcemanager.connect.* settings. When not set,
yarn.resourcemanager.connect.retry-interval.ms is used instead.</description>
<name>yarn.client.failover-sleep-max-ms</name>
<!--value>15000</value-->
</property>
<property>
<description>When HA is enabled, the number of retries per
attempt to connect to a ResourceManager. In other words,
it is the ipc.client.connect.max.retries to be used during
failover attempts</description>
<name>yarn.client.failover-retries</name>
<value>0</value>
</property>
<property>
<description>When HA is enabled, the number of retries per
attempt to connect to a ResourceManager on socket timeouts. In other
words, it is the ipc.client.connect.max.retries.on.timeouts to be used
during failover attempts</description>
<name>yarn.client.failover-retries-on-socket-timeouts</name>
<value>0</value>
</property>
<property>
<description>The maximum number of completed applications RM keeps. </description>
<name>yarn.resourcemanager.max-completed-applications</name>
<value>1000</value>
</property>
<property>
<description>Interval at which the delayed token removal thread runs</description>
<name>yarn.resourcemanager.delayed.delegation-token.removal-interval-ms</name>
<value>30000</value>
</property>
<property>
<description>Maximum size in bytes for configurations that can be provided
by application to RM for delegation token renewal.
By experiment, it's roughly 128 bytes per key-value pair.
The default value 12800 allows roughly 100 configs, may be less.
</description>
<name>yarn.resourcemanager.delegation-token.max-conf-size-bytes</name>
<value>12800</value>
</property>
<property>
<description>If true, ResourceManager will have proxy-user privileges.
Use case: In a secure cluster, YARN requires the user hdfs delegation-tokens to
do localization and log-aggregation on behalf of the user. If this is set to true,
ResourceManager is able to request new hdfs delegation tokens on behalf of
the user. This is needed by long-running-service, because the hdfs tokens
will eventually expire and YARN requires new valid tokens to do localization
and log-aggregation. Note that to enable this use case, the corresponding
HDFS NameNode has to configure ResourceManager as the proxy-user so that
ResourceManager can itself ask for new tokens on behalf of the user when
tokens are past their max-life-time.</description>
<name>yarn.resourcemanager.proxy-user-privileges.enabled</name>
<value>false</value>
</property>
<property>
<description>Interval for the roll over for the master key used to generate
application tokens
</description>
<name>yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs</name>
<value>86400</value>
</property>
<property>
<description>Interval for the roll over for the master key used to generate
container tokens. It is expected to be much greater than
yarn.nm.liveness-monitor.expiry-interval-ms and
yarn.resourcemanager.rm.container-allocation.expiry-interval-ms. Otherwise the
behavior is undefined.
</description>
<name>yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs</name>
<value>86400</value>
</property>
<property>
<description>The heart-beat interval in milliseconds for every NodeManager in the cluster.</description>
<name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
<value>1000</value>
</property>
<property>
<description>The minimum allowed version of a connecting nodemanager. The valid values are
NONE (no version checking), EqualToRM (the nodemanager's version is equal to
or greater than the RM version), or a Version String.</description>
<name>yarn.resourcemanager.nodemanager.minimum.version</name>
<value>NONE</value>
</property>
<property>
<description>Enable a set of periodic monitors (specified in
yarn.resourcemanager.scheduler.monitor.policies) that affect the
scheduler.</description>
<name>yarn.resourcemanager.scheduler.monitor.enable</name>
<value>false</value>
</property>
<property>
<description>The list of SchedulingEditPolicy classes that interact with
the scheduler. A particular module may be incompatible with the
scheduler, other policies, or a configuration of either.</description>
<name>yarn.resourcemanager.scheduler.monitor.policies</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy</value>
</property>
<property>
<description>The class to use as the configuration provider.
If org.apache.hadoop.yarn.LocalConfigurationProvider is used,
the local configuration will be loaded.
If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used,
the configuration which will be loaded should be uploaded to remote File system first.
</description>
<name>yarn.resourcemanager.configuration.provider-class</name>
<value>org.apache.hadoop.yarn.LocalConfigurationProvider</value>
<!-- <value>org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider</value> -->
</property>
<property>
<description>
The value specifies the file system (e.g. HDFS) path where ResourceManager
loads configuration if yarn.resourcemanager.configuration.provider-class
is set to org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider.
</description>
<name>yarn.resourcemanager.configuration.file-system-based-store</name>
<value>/yarn/conf</value>
</property>
<property>
<description>The setting that controls whether yarn system metrics is
published to the Timeline server (version one) or not, by RM.
This configuration is now deprecated in favor of
yarn.system-metrics-publisher.enabled.</description>
<name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
<property>
<description>The setting that controls whether yarn system metrics is
published on the Timeline service or not by RM And NM.</description>
<name>yarn.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
<property>
<description>The setting that controls whether yarn container events are
published to the timeline service or not by RM. This configuration setting
is for ATS V2.</description>
<name>yarn.rm.system-metrics-publisher.emit-container-events</name>
<value>false</value>
</property>
<property>
<description>Number of worker threads that send the yarn system metrics
data.</description>
<name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
<value>10</value>
</property>
<property>
<description>Number of diagnostics/failure messages can be saved in RM for
log aggregation. It also defines the number of diagnostics/failure
messages can be shown in log aggregation web ui.</description>
<name>yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory</name>
<value>10</value>
</property>
<!-- Node Manager Configs -->
<property>
<description>
RM DelegationTokenRenewer thread count
</description>
<name>yarn.resourcemanager.delegation-token-renewer.thread-count</name>
<value>50</value>
</property>
<property>
<description>
RM secret key update interval in ms
</description>
<name>yarn.resourcemanager.delegation.key.update-interval</name>
<value>86400000</value>
</property>
<property>
<description>
RM delegation token maximum lifetime in ms
</description>
<name>yarn.resourcemanager.delegation.token.max-lifetime</name>
<value>604800000</value>
</property>
<property>
<description>
RM delegation token update interval in ms
</description>
<name>yarn.resourcemanager.delegation.token.renew-interval</name>
<value>86400000</value>
</property>
<property>
<description>
Thread pool size for RMApplicationHistoryWriter.
</description>
<name>yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size</name>
<value>10</value>
</property>
<property>
<description>
Comma-separated list of values (in minutes) for schedule queue related
metrics.
</description>
<name>yarn.resourcemanager.metrics.runtime.buckets</name>
<value>60,300,1440</value>
</property>
<property>
<description>
Interval for the roll over for the master key used to generate
NodeManager tokens. It is expected to be set to a value much larger
than yarn.nm.liveness-monitor.expiry-interval-ms.
</description>
<name>yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs</name>
<value>86400</value>
</property>
<property>
<description>
Flag to enable the ResourceManager reservation system.
</description>
<name>yarn.resourcemanager.reservation-system.enable</name>
<value>false</value>
</property>
<property>
<description>
The Java class to use as the ResourceManager reservation system.
By default, is set to
org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityReservationSystem
when using CapacityScheduler and is set to
org.apache.hadoop.yarn.server.resourcemanager.reservation.FairReservationSystem
when using FairScheduler.
</description>
<name>yarn.resourcemanager.reservation-system.class</name>
<value></value>
</property>
<property>
<description>
The plan follower policy class name to use for the ResourceManager
reservation system.
By default, is set to
org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacitySchedulerPlanFollower
is used when using CapacityScheduler, and is set to
org.apache.hadoop.yarn.server.resourcemanager.reservation.FairSchedulerPlanFollower
when using FairScheduler.
</description>
<name>yarn.resourcemanager.reservation-system.plan.follower</name>
<value></value>
</property>
<property>
<description>
Step size of the reservation system in ms
</description>
<name>yarn.resourcemanager.reservation-system.planfollower.time-step</name>
<value>1000</value>
</property>
<property>
<description>
The expiry interval for a container
</description>
<name>yarn.resourcemanager.rm.container-allocation.expiry-interval-ms</name>
<value>600000</value>
</property>
<property>
<description>
Flag to enable/disable resource profiles
</description>
<name>yarn.resourcemanager.resource-profiles.enabled</name>
<value>false</value>
</property>
<property>
<description>
If resource profiles is enabled, source file for the profiles
</description>
<name>yarn.resourcemanager.resource-profiles.source-file</name>
<value>resource-profiles.json</value>
</property>
<!-- Node Manager Configuration -->
<property>
<description>The hostname of the NM.</description>
<name>yarn.nodemanager.hostname</name>
<value>0.0.0.0</value>
</property>
<property>
<description>The address of the container manager in the NM.</description>
<name>yarn.nodemanager.address</name>
<value>${yarn.nodemanager.hostname}:0</value>
</property>
<property>
<description>
The actual address the server will bind to. If this optional address is
set, the RPC and webapp servers will bind to this address and the port specified in
yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively. This is
most useful for making NM listen to all interfaces by setting to 0.0.0.0.
</description>
<name>yarn.nodemanager.bind-host</name>
<value></value>
</property>
<property>
<description>Environment variables that should be forwarded from the NodeManager's environment to the container's.</description>
<name>yarn.nodemanager.admin-env</name>
<value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
</property>
<property>
<description>Environment variables that containers may override rather than use NodeManager's default.</description>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ</value>
</property>
<property>
<description>who will execute(launch) the containers.</description>
<name>yarn.nodemanager.container-executor.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
</property>
<property>
<description>Comma separated List of container state transition listeners.</description>
<name>yarn.nodemanager.container-state-transition-listener.classes</name>
<value></value>
</property>
<property>
<description>Number of threads container manager uses.</description>
<name>yarn.nodemanager.container-manager.thread-count</name>
<value>20</value>
</property>
<property>
<description>Number of threads collector service uses.</description>
<name>yarn.nodemanager.collector-service.thread-count</name>
<value>5</value>
</property>
<property>
<description>Number of threads used in cleanup.</description>
<name>yarn.nodemanager.delete.thread-count</name>
<value>4</value>
</property>
<property>
<description>Max number of OPPORTUNISTIC containers to queue at the
nodemanager.</description>
<name>yarn.nodemanager.opportunistic-containers-max-queue-length</name>
<value>0</value>
</property>
<property>
<description>
Number of seconds after an application finishes before the nodemanager's
DeletionService will delete the application's localized file directory
and log directory.
To diagnose YARN application problems, set this property's value large
enough (for example, to 600 = 10 minutes) to permit examination of these
directories. After changing the property's value, you must restart the
nodemanager in order for it to have an effect.
The roots of YARN applications' work directories is configurable with
the yarn.nodemanager.local-dirs property (see below), and the roots
of the YARN applications' log directories is configurable with the
yarn.nodemanager.log-dirs property (see also below).
</description>
<name>yarn.nodemanager.delete.debug-delay-sec</name>
<value>0</value>
</property>
<property>
<description>Keytab for NM.</description>
<name>yarn.nodemanager.keytab</name>
<value>/etc/krb5.keytab</value>
</property>
<property>
<description>List of directories to store localized files in. An
application's localized file directory will be found in:
${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}.
Individual containers' work directories, called container_${contid}, will
be subdirectories of this.
</description>
<name>yarn.nodemanager.local-dirs</name>
<value>${hadoop.tmp.dir}/nm-local-dir</value>
</property>
<property>
<description>It limits the maximum number of files which will be localized
in a single local directory. If the limit is reached then sub-directories
will be created and new files will be localized in them. If it is set to
a value less than or equal to 36 [which are sub-directories (0-9 and then
a-z)] then NodeManager will fail to start. For example; [for public
cache] if this is configured with a value of 40 ( 4 files +
36 sub-directories) and the local-dir is "/tmp/local-dir1" then it will
allow 4 files to be created directly inside "/tmp/local-dir1/filecache".
For files that are localized further it will create a sub-directory "0"
inside "/tmp/local-dir1/filecache" and will localize files inside it
until it becomes full. If a file is removed from a sub-directory that
is marked full, then that sub-directory will be used back again to
localize files.
</description>
<name>yarn.nodemanager.local-cache.max-files-per-directory</name>
<value>8192</value>
</property>
<property>
<description>Address where the localizer IPC is.</description>
<name>yarn.nodemanager.localizer.address</name>
<value>${yarn.nodemanager.hostname}:8040</value>
</property>
<property>
<description>Address where the collector service IPC is.</description>
<name>yarn.nodemanager.collector-service.address</name>
<value>${yarn.nodemanager.hostname}:8048</value>
</property>
<property>
<description>Interval in between cache cleanups.</description>
<name>yarn.nodemanager.localizer.cache.cleanup.interval-ms</name>
<value>600000</value>
</property>
<property>
<description>Target size of localizer cache in MB, per nodemanager. It is
a target retention size that only includes resources with PUBLIC and
PRIVATE visibility and excludes resources with APPLICATION visibility
</description>
<name>yarn.nodemanager.localizer.cache.target-size-mb</name>
<value>10240</value>
</property>
<property>
<description>Number of threads to handle localization requests.</description>
<name>yarn.nodemanager.localizer.client.thread-count</name>
<value>5</value>
</property>
<property>
<description>Number of threads to use for localization fetching.</description>
<name>yarn.nodemanager.localizer.fetch.thread-count</name>
<value>4</value>
</property>
<property>
<description>
</description>
<name>yarn.nodemanager.container-localizer.java.opts</name>
<value>-Xmx256m</value>
</property>
<property>
<description>
The log level for container localizer while it is an independent process.
</description>
<name>yarn.nodemanager.container-localizer.log.level</name>
<value>INFO</value>
</property>
<property>
<description>
Where to store container logs. An application's localized log directory
will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
Individual containers' log directories will be below this, in directories
named container_{$contid}. Each container directory will contain the files
stderr, stdin, and syslog generated by that container.
</description>
<name>yarn.nodemanager.log-dirs</name>
<value>${yarn.log.dir}/userlogs</value>
</property>
<property>
<description>
The permissions settings used for the creation of container
directories when using DefaultContainerExecutor. This follows
standard user/group/all permissions format.
</description>
<name>yarn.nodemanager.default-container-executor.log-dirs.permissions</name>
<value>710</value>
</property>
<property>
<description>Whether to enable log aggregation. Log aggregation collects
each container's logs and moves these logs onto a file-system, for e.g.
HDFS, after the application completes. Users can configure the
"yarn.nodemanager.remote-app-log-dir" and
"yarn.nodemanager.remote-app-log-dir-suffix" properties to determine
where these logs are moved to. Users can access the logs via the
Application Timeline Server.
</description>
<name>yarn.log-aggregation-enable</name>
<value>false</value>
</property>
<property>
<description>How long to keep aggregation logs before deleting them. -1 disables.
Be careful set this too small and you will spam the name node.</description>
<name>yarn.log-aggregation.retain-seconds</name>
<value>-1</value>
</property>
<property>
<description>How long to wait between aggregated log retention checks.
If set to 0 or a negative value then the value is computed as one-tenth
of the aggregated log retention time. Be careful set this too small and
you will spam the name node.</description>
<name>yarn.log-aggregation.retain-check-interval-seconds</name>
<value>-1</value>
</property>
<property>
<description>Specify which log file controllers we will support. The first
file controller we add will be used to write the aggregated logs.
This comma separated configuration will work with the configuration:
yarn.log-aggregation.file-controller.%s.class which defines the supported
file controller's class. By default, the TFile controller would be used.
The user could override this configuration by adding more file controllers.
To support back-ward compatibility, make sure that we always
add TFile file controller.</description>
<name>yarn.log-aggregation.file-formats</name>
<value>TFile</value>
</property>
<property>
<description>Class that supports TFile read and write operations.</description>
<name>yarn.log-aggregation.file-controller.TFile.class</name>
<value>org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController</value>
</property>
<property>
<description>
How long for ResourceManager to wait for NodeManager to report its
log aggregation status. If waiting time of which the log aggregation
status is reported from NodeManager exceeds the configured value, RM
will report log aggregation status for this NodeManager as TIME_OUT.
This configuration will be used in NodeManager as well to decide
whether and when to delete the cached log aggregation status.
</description>
<name>yarn.log-aggregation-status.time-out.ms</name>
<value>600000</value>
</property>
<property>
<description>Time in seconds to retain user logs. Only applicable if
log aggregation is disabled
</description>
<name>yarn.nodemanager.log.retain-seconds</name>
<value>10800</value>
</property>
<property>
<description>Where to aggregate logs to.</description>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/tmp/logs</value>
</property>
<property>
<description>The remote log dir will be created at
{yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}
</description>
<name>yarn.nodemanager.remote-app-log-dir-suffix</name>
<value>logs</value>
</property>
<property>
<description>Generate additional logs about container launches.
Currently, this creates a copy of the launch script and lists the
directory contents of the container work dir. When listing directory
contents, we follow symlinks to a max-depth of 5(including symlinks
which point to outside the container work dir) which may lead to a
slowness in launching containers.
</description>
<name>yarn.nodemanager.log-container-debug-info.enabled</name>
<value>true</value>
</property>
<property>
<description>Amount of physical memory, in MB, that can be allocated
for containers. If set to -1 and
yarn.nodemanager.resource.detect-hardware-capabilities is true, it is
automatically calculated(in case of Windows and Linux).
In other cases, the default is 8192MB.
</description>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>-1</value>
</property>
<property>
<description>Amount of physical memory, in MB, that is reserved
for non-YARN processes. This configuration is only used if
yarn.nodemanager.resource.detect-hardware-capabilities is set
to true and yarn.nodemanager.resource.memory-mb is -1. If set
to -1, this amount is calculated as
20% of (system memory - 2*HADOOP_HEAPSIZE)
</description>
<name>yarn.nodemanager.resource.system-reserved-memory-mb</name>
<value>-1</value>
</property>
<property>
<description>Whether YARN CGroups memory tracking is enabled.</description>
<name>yarn.nodemanager.resource.memory.enabled</name>
<value>false</value>
</property>
<property>
<description>Whether YARN CGroups strict memory enforcement is enabled.
</description>
<name>yarn.nodemanager.resource.memory.enforced</name>
<value>true</value>
</property>
<property>
<description>If memory limit is enforced, this the percentage of soft limit
compared to the memory assigned to the container. If there is memory
pressure container memory usage will be pushed back to its soft limit
by swapping out memory.
</description>
<name>yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage</name>
<value>90.0</value>
</property>
<property>
<description>Container swappiness is the likelihood a page will be swapped
out compared to be kept in memory. Value is between 0-100.
</description>
<name>yarn.nodemanager.resource.memory.cgroups.swappiness</name>
<value>0</value>
</property>
<property>
<description>Whether physical memory limits will be enforced for
containers.</description>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>true</value>
</property>
<property>
<description>Whether virtual memory limits will be enforced for
containers.</description>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>true</value>
</property>
<property>
<description>Ratio between virtual memory to physical memory when
setting memory limits for containers. Container allocations are
expressed in terms of physical memory, and virtual memory usage
is allowed to exceed this allocation by this ratio.
</description>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>2.1</value>
</property>
<property>
<description>Number of vcores that can be allocated
for containers. This is used by the RM scheduler when allocating
resources for containers. This is not used to limit the number of
CPUs used by YARN containers. If it is set to -1 and
yarn.nodemanager.resource.detect-hardware-capabilities is true, it is
automatically determined from the hardware in case of Windows and Linux.
In other cases, number of vcores is 8 by default.</description>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>-1</value>
</property>
<property>
<description>Flag to determine if logical processors(such as
hyperthreads) should be counted as cores. Only applicable on Linux
when yarn.nodemanager.resource.cpu-vcores is set to -1 and
yarn.nodemanager.resource.detect-hardware-capabilities is true.
</description>
<name>yarn.nodemanager.resource.count-logical-processors-as-cores</name>
<value>false</value>
</property>
<property>
<description>Multiplier to determine how to convert phyiscal cores to
vcores. This value is used if yarn.nodemanager.resource.cpu-vcores
is set to -1(which implies auto-calculate vcores) and
yarn.nodemanager.resource.detect-hardware-capabilities is set to true. The
number of vcores will be calculated as
number of CPUs * multiplier.
</description>
<name>yarn.nodemanager.resource.pcores-vcores-multiplier</name>
<value>1.0</value>
</property>
<property>
<description>
Thread pool size for LogAggregationService in Node Manager.
</description>
<name>yarn.nodemanager.logaggregation.threadpool-size-max</name>
<value>100</value>
</property>
<property>
<description>Percentage of CPU that can be allocated
for containers. This setting allows users to limit the amount of
CPU that YARN containers use. Currently functional only
on Linux using cgroups. The default is to use 100% of CPU.
</description>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
<value>100</value>
</property>
<property>
<description>Enable auto-detection of node capabilities such as
memory and CPU.
</description>
<name>yarn.nodemanager.resource.detect-hardware-capabilities</name>
<value>false</value>
</property>
<property>
<description>NM Webapp address.</description>
<name>yarn.nodemanager.webapp.address</name>
<value>${yarn.nodemanager.hostname}:8042</value>
</property>
<property>
<description>
The https adddress of the NM web application.
</description>
<name>yarn.nodemanager.webapp.https.address</name>
<value>0.0.0.0:8044</value>
</property>
<property>
<description>
The Kerberos keytab file to be used for spnego filter for the NM web
interface.
</description>
<name>yarn.nodemanager.webapp.spnego-keytab-file</name>
<value></value>
</property>
<property>
<description>
The Kerberos principal to be used for spnego filter for the NM web
interface.
</description>
<name>yarn.nodemanager.webapp.spnego-principal</name>
<value></value>
</property>
<property>
<description>How often to monitor the node and the containers.
If 0 or negative, monitoring is disabled.</description>
<name>yarn.nodemanager.resource-monitor.interval-ms</name>
<value>3000</value>
</property>
<property>
<description>Class that calculates current resource utilization.</description>
<name>yarn.nodemanager.resource-calculator.class</name>
</property>
<property>
<description>Enable container monitor</description>
<name>yarn.nodemanager.container-monitor.enabled</name>
<value>true</value>
</property>
<property>
<description>How often to monitor containers. If not set, the value for
yarn.nodemanager.resource-monitor.interval-ms will be used.
If 0 or negative, container monitoring is disabled.</description>
<name>yarn.nodemanager.container-monitor.interval-ms</name>
</property>
<property>
<description>Class that calculates containers current resource utilization.
If not set, the value for yarn.nodemanager.resource-calculator.class will
be used.</description>
<name>yarn.nodemanager.container-monitor.resource-calculator.class</name>
</property>
<property>
<description>Frequency of running node health script.</description>
<name>yarn.nodemanager.health-checker.interval-ms</name>
<value>600000</value>
</property>
<property>
<description>Script time out period.</description>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
<value>1200000</value>
</property>
<property>
<description>The health check script to run.</description>
<name>yarn.nodemanager.health-checker.script.path</name>
<value></value>
</property>
<property>
<description>The arguments to pass to the health check script.</description>
<name>yarn.nodemanager.health-checker.script.opts</name>
<value></value>
</property>
<property>
<description>Frequency of running disk health checker code.</description>
<name>yarn.nodemanager.disk-health-checker.interval-ms</name>
<value>120000</value>
</property>
<property>
<description>The minimum fraction of number of disks to be healthy for the
nodemanager to launch new containers. This correspond to both
yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e. If there
are less number of healthy local-dirs (or log-dirs) available, then
new containers will not be launched on this node.</description>
<name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
<value>0.25</value>
</property>
<property>
<description>The maximum percentage of disk space utilization allowed after
which a disk is marked as bad. Values can range from 0.0 to 100.0.
If the value is greater than or equal to 100, the nodemanager will check
for full disk. This applies to yarn.nodemanager.local-dirs and
yarn.nodemanager.log-dirs.</description>
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
<value>90.0</value>
</property>
<property>
<description>The low threshold percentage of disk space used when a bad disk is
marked as good. Values can range from 0.0 to 100.0. This applies to
yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs.
Note that if its value is more than yarn.nodemanager.disk-health-checker.
max-disk-utilization-per-disk-percentage or not set, it will be set to the same value as
yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage.</description>
<name>yarn.nodemanager.disk-health-checker.disk-utilization-watermark-low-per-disk-percentage</name>
<value></value>
</property>
<property>
<description>The minimum space that must be available on a disk for
it to be used. This applies to yarn.nodemanager.local-dirs and
yarn.nodemanager.log-dirs.</description>
<name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
<value>0</value>
</property>
<property>
<description>The path to the Linux container executor.</description>
<name>yarn.nodemanager.linux-container-executor.path</name>
</property>
<property>
<description>The class which should help the LCE handle resources.</description>
<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
<!-- <value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value> -->
</property>
<property>
<description>The cgroups hierarchy under which to place YARN proccesses (cannot contain commas).
If yarn.nodemanager.linux-container-executor.cgroups.mount is false
(that is, if cgroups have been pre-configured) and the YARN user has write
access to the parent directory, then the directory will be created.
If the directory already exists, the administrator has to give YARN
write permissions to it recursively.
This property only applies when the LCE resources handler is set to
CgroupsLCEResourcesHandler.</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
<value>/hadoop-yarn</value>
</property>
<property>
<description>Whether the LCE should attempt to mount cgroups if not found.
This property only applies when the LCE resources handler is set to
CgroupsLCEResourcesHandler.
</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
<value>false</value>
</property>
<property>
<description>This property sets the path from which YARN will read the
CGroups configuration. YARN has built-in functionality to discover the
system CGroup mount paths, so use this property only if YARN's automatic
mount path discovery does not work.
The path specified by this property must exist before the NodeManager is
launched.
If yarn.nodemanager.linux-container-executor.cgroups.mount is set to true,
YARN will first try to mount the CGroups at the specified path before
reading them.
If yarn.nodemanager.linux-container-executor.cgroups.mount is set to
false, YARN will read the CGroups at the specified path.
If this property is empty, YARN tries to detect the CGroups location.
Please refer to NodeManagerCgroups.html in the documentation for further
details.
This property only applies when the LCE resources handler is set to
CgroupsLCEResourcesHandler.
</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
</property>
<property>
<description>Delay in ms between attempts to remove linux cgroup</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms</name>
<value>20</value>
</property>
<property>
<description>This determines which of the two modes that LCE should use on
a non-secure cluster. If this value is set to true, then all containers
will be launched as the user specified in
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user. If
this value is set to false, then containers will run as the user who
submitted the application.</description>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
<value>true</value>
</property>
<property>
<description>The UNIX user that containers will run as when
Linux-container-executor is used in nonsecure mode (a use case for this
is using cgroups) if the
yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is
set to true.</description>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user</name>
<value>nobody</value>
</property>
<property>
<description>The allowed pattern for UNIX user names enforced by
Linux-container-executor when used in nonsecure mode (use case for this
is using cgroups). The default value is taken from /usr/sbin/adduser</description>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern</name>
<value>^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}[$]$</value>
</property>
<property>
<description>This flag determines whether apps should run with strict resource limits
or be allowed to consume spare resources if they need them. For example, turning the
flag on will restrict apps to use only their share of CPU, even if the node has spare
CPU cycles. The default value is false i.e. use available resources. Please note that
turning this flag on may reduce job throughput on the cluster. This setting does
not apply to other subsystems like memory.</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
<value>false</value>
</property>
<property>
<description>Comma separated list of runtimes that are allowed when using
LinuxContainerExecutor. The allowed values are default, docker, and
javasandbox.</description>
<name>yarn.nodemanager.runtime.linux.allowed-runtimes</name>
<value>default</value>
</property>
<property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
uppercase. To run without any capabilites, set this value to
"none" or "NONE"</description>
<name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
<value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
</property>
<property>
<description>This configuration setting determines if
privileged docker containers are allowed on this cluster.
Use with extreme care.</description>
<name>yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed</name>
<value>false</value>
</property>
<property>
<description>This configuration setting determines who is allowed to run
privileged docker containers on this cluster. Use with extreme care.
</description>
<name>yarn.nodemanager.runtime.linux.docker.privileged-containers.acl</name>
<value></value>
</property>
<property>
<description>The set of networks allowed when launching containers using the
DockerContainerRuntime.</description>
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
<value>host,none,bridge</value>
</property>
<property>
<description>The network used when launching containers using the
DockerContainerRuntime when no network is specified in the request
. This network must be one of the (configurable) set of allowed container
networks.</description>
<name>yarn.nodemanager.runtime.linux.docker.default-container-network</name>
<value>host</value>
</property>
<property>
<description>This configuration setting determines whether the host's PID
namespace is allowed for docker containers on this cluster.
Use with care.</description>
<name>yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed</name>
<value>false</value>
</property>
<property>
<description>Property to enable docker user remapping</description>
<name>yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed</name>
<value>true</value>
</property>
<property>
<description>lower limit for acceptable uids of user remapped user</description>
<name>yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold</name>
<value>1</value>
</property>
<property>
<description>lower limit for acceptable gids of user remapped user</description>
<name>yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold</name>
<value>1</value>
</property>
<property>
<description>Whether or not users are allowed to request that Docker
containers honor the debug deletion delay. This is useful for
troubleshooting Docker container related launch failures.</description>
<name>yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed</name>
<value>false</value>
</property>
<property>
<description>The default list of read-only mounts to be bind-mounted
into all Docker containers that use DockerContainerRuntime.</description>
<name>yarn.nodemanager.runtime.linux.docker.default-ro-mounts</name>
<value></value>
</property>
<property>
<description>The default list of read-write mounts to be bind-mounted
into all Docker containers that use DockerContainerRuntime.</description>
<name>yarn.nodemanager.runtime.linux.docker.default-rw-mounts</name>
<value></value>
</property>
<property>
<description>The mode in which the Java Container Sandbox should run detailed by
the JavaSandboxLinuxContainerRuntime.</description>
<name>yarn.nodemanager.runtime.linux.sandbox-mode</name>
<value>disabled</value>
</property>
<property>
<description>Permissions for application local directories.</description>
<name>yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions</name>
<value>read</value>
</property>
<property>
<description>Location for non-default java policy file.</description>
<name>yarn.nodemanager.runtime.linux.sandbox-mode.policy</name>
<value></value>
</property>
<property>
<description>The group which will run by default without the java security
manager.</description>
<name>yarn.nodemanager.runtime.linux.sandbox-mode.whitelist-group</name>
<value></value>
</property>
<property>
<description>This flag determines whether memory limit will be set for the Windows Job
Object of the containers launched by the default container executor.</description>
<name>yarn.nodemanager.windows-container.memory-limit.enabled</name>
<value>false</value>
</property>
<property>
<description>This flag determines whether CPU limit will be set for the Windows Job
Object of the containers launched by the default container executor.</description>
<name>yarn.nodemanager.windows-container.cpu-limit.enabled</name>
<value>false</value>
</property>
<property>
<description>
Interval of time the linux container executor should try cleaning up
cgroups entry when cleaning up a container.
</description>
<name>yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms</name>
<value>1000</value>
</property>
<property>
<description>
The UNIX group that the linux-container-executor should run as.
</description>
<name>yarn.nodemanager.linux-container-executor.group</name>
<value></value>
</property>
<property>
<description>T-file compression types used to compress aggregated logs.</description>
<name>yarn.nodemanager.log-aggregation.compression-type</name>
<value>none</value>
</property>
<property>
<description>The kerberos principal for the node manager.</description>
<name>yarn.nodemanager.principal</name>
<value></value>
</property>
<property>
<description>A comma separated list of services where service name should only
contain a-zA-Z0-9_ and can not start with numbers</description>
<name>yarn.nodemanager.aux-services</name>
<value></value>
<!--<value>mapreduce_shuffle</value>-->
</property>
<property>
<description>No. of ms to wait between sending a SIGTERM and SIGKILL to a container</description>
<name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
<value>250</value>
</property>
<property>
<description>Max time to wait for a process to come up when trying to cleanup a container</description>
<name>yarn.nodemanager.process-kill-wait.ms</name>
<value>5000</value>
</property>
<property>
<description>The minimum allowed version of a resourcemanager that a nodemanager will connect to.
The valid values are NONE (no version checking), EqualToNM (the resourcemanager's version is
equal to or greater than the NM version), or a Version String.</description>
<name>yarn.nodemanager.resourcemanager.minimum.version</name>
<value>NONE</value>
</property>
<property>
<description>Maximum size of contain's diagnostics to keep for relaunching
container case.</description>
<name>yarn.nodemanager.container-diagnostics-maximum-size</name>
<value>10000</value>
</property>
<property>
<description>Minimum container restart interval in milliseconds.</description>
<name>yarn.nodemanager.container-retry-minimum-interval-ms</name>
<value>1000</value>
</property>
<property>
<description>Max number of threads in NMClientAsync to process container
management events</description>
<name>yarn.client.nodemanager-client-async.thread-pool-max-size</name>
<value>500</value>
</property>
<property>
<description>Max time to wait to establish a connection to NM</description>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
<value>180000</value>
</property>
<property>
<description>Time interval between each attempt to connect to NM</description>
<name>yarn.client.nodemanager-connect.retry-interval-ms</name>
<value>10000</value>
</property>
<property>
<description>
Max time to wait for NM to connect to RM.
When not set, proxy will fall back to use value of
yarn.resourcemanager.connect.max-wait.ms.
</description>
<name>yarn.nodemanager.resourcemanager.connect.max-wait.ms</name>
<value></value>
</property>
<property>
<description>
Time interval between each NM attempt to connect to RM.
When not set, proxy will fall back to use value of
yarn.resourcemanager.connect.retry-interval.ms.
</description>
<name>yarn.nodemanager.resourcemanager.connect.retry-interval.ms</name>
<value></value>
</property>
<property>
<description>
Maximum number of proxy connections to cache for node managers. If set
to a value greater than zero then the cache is enabled and the NMClient
and MRAppMaster will cache the specified number of node manager proxies.
There will be at max one proxy per node manager. Ex. configuring it to a
value of 5 will make sure that client will at max have 5 proxies cached
with 5 different node managers. These connections for these proxies will
be timed out if idle for more than the system wide idle timeout period.
Note that this could cause issues on large clusters as many connections
could linger simultaneously and lead to a large number of connection
threads. The token used for authentication will be used only at
connection creation time. If a new token is received then the earlier
connection should be closed in order to use the new token. This and
(yarn.client.nodemanager-client-async.thread-pool-max-size) are related
and should be in sync (no need for them to be equal).
If the value of this property is zero then the connection cache is
disabled and connections will use a zero idle timeout to prevent too
many connection threads on large clusters.
</description>
<name>yarn.client.max-cached-nodemanagers-proxies</name>
<value>0</value>
</property>
<property>
<description>Enable the node manager to recover after starting</description>
<name>yarn.nodemanager.recovery.enabled</name>
<value>false</value>
</property>
<property>
<description>The local filesystem directory in which the node manager will
store state when recovery is enabled.</description>
<name>yarn.nodemanager.recovery.dir</name>
<value>${hadoop.tmp.dir}/yarn-nm-recovery</value>
</property>
<property>
<description>The time in seconds between full compactions of the NM state
database. Setting the interval to zero disables the full compaction
cycles.</description>
<name>yarn.nodemanager.recovery.compaction-interval-secs</name>
<value>3600</value>
</property>
<property>
<description>Whether the nodemanager is running under supervision. A
nodemanager that supports recovery and is running under supervision
will not try to cleanup containers as it exits with the assumption
it will be immediately be restarted and recover containers.</description>
<name>yarn.nodemanager.recovery.supervised</name>
<value>false</value>
</property>
<!--Docker configuration-->
<property>
<description>
Adjustment to the container OS scheduling priority. In Linux, passed
directly to the nice command. If unspecified then containers are launched
without any explicit OS priority.
</description>
<name>yarn.nodemanager.container-executor.os.sched.priority.adjustment</name>
</property>
<property>
<description>
Flag to enable container metrics
</description>
<name>yarn.nodemanager.container-metrics.enable</name>
<value>true</value>
</property>
<property>
<description>
Container metrics flush period in ms. Set to -1 for flush on completion.
</description>
<name>yarn.nodemanager.container-metrics.period-ms</name>
<value>-1</value>
</property>
<property>
<description>
The delay time ms to unregister container metrics after completion.
</description>
<name>yarn.nodemanager.container-metrics.unregister-delay-ms</name>
<value>10000</value>
</property>
<property>
<description>
Class used to calculate current container resource utilization.
</description>
<name>yarn.nodemanager.container-monitor.process-tree.class</name>
<value></value>
</property>
<property>
<description>
Flag to enable NodeManager disk health checker
</description>
<name>yarn.nodemanager.disk-health-checker.enable</name>
<value>true</value>
</property>
<property>
<description>
Number of threads to use in NM log cleanup. Used when log aggregation
is disabled.
</description>
<name>yarn.nodemanager.log.deletion-threads-count</name>
<value>4</value>
</property>
<property>
<description>
The Windows group that the windows-container-executor should run as.
</description>
<name>yarn.nodemanager.windows-secure-container-executor.group</name>
<value></value>
</property>
<!-- Map Reduce Configuration -->
<property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<!-- WebAppProxy Configuration -->
<property>
<description>The kerberos principal for the proxy, if the proxy is not
running as part of the RM.</description>
<name>yarn.web-proxy.principal</name>
<value/>
</property>
<property>
<description>Keytab for WebAppProxy, if the proxy is not running as part of
the RM.</description>
<name>yarn.web-proxy.keytab</name>
</property>
<property>
<description>The address for the web proxy as HOST:PORT, if this is not
given then the proxy will run as part of the RM</description>
<name>yarn.web-proxy.address</name>
<value/>
</property>
<!-- Applications' Configuration -->
<property>
<description>
CLASSPATH for YARN applications. A comma-separated list
of CLASSPATH entries. When this value is empty, the following default
CLASSPATH for YARN applications would be used.
For Linux:
$HADOOP_CONF_DIR,
$HADOOP_COMMON_HOME/share/hadoop/common/*,
$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,
$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,
$HADOOP_YARN_HOME/share/hadoop/yarn/*,
$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*
For Windows:
%HADOOP_CONF_DIR%,
%HADOOP_COMMON_HOME%/share/hadoop/common/*,
%HADOOP_COMMON_HOME%/share/hadoop/common/lib/*,
%HADOOP_HDFS_HOME%/share/hadoop/hdfs/*,
%HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*,
%HADOOP_YARN_HOME%/share/hadoop/yarn/*,
%HADOOP_YARN_HOME%/share/hadoop/yarn/lib/*
</description>
<name>yarn.application.classpath</name>
<value></value>
</property>
<!-- Timeline Service Configuration -->
<property>
<description>Indicate what is the current version of the running
timeline service. For example, if "yarn.timeline-service.version" is 1.5,
and "yarn.timeline-service.enabled" is true, it means the cluster will and
should bring up the timeline service v.1.5 (and nothing else).
On the client side, if the client uses the same version of timeline service,
it should succeed. If the client chooses to use a smaller version in spite of this,
then depending on how robust the compatibility story is between versions,
the results may vary.
</description>
<name>yarn.timeline-service.version</name>
<value>1.0f</value>
</property>
<property>
<description>
In the server side it indicates whether timeline service is enabled or not.
And in the client side, users can enable it to indicate whether client wants
to use timeline service. If it's enabled in the client side along with
security, then yarn client tries to fetch the delegation tokens for the
timeline server.
</description>
<name>yarn.timeline-service.enabled</name>
<value>false</value>
</property>
<property>
<description>The hostname of the timeline service web application.</description>
<name>yarn.timeline-service.hostname</name>
<value>0.0.0.0</value>
</property>
<property>
<description>This is default address for the timeline server to start the
RPC server.</description>
<name>yarn.timeline-service.address</name>
<value>${yarn.timeline-service.hostname}:10200</value>
</property>
<property>
<description>The http address of the timeline service web application.</description>
<name>yarn.timeline-service.webapp.address</name>
<value>${yarn.timeline-service.hostname}:8188</value>
</property>
<property>
<description>The https address of the timeline service web application.</description>
<name>yarn.timeline-service.webapp.https.address</name>
<value>${yarn.timeline-service.hostname}:8190</value>
</property>
<property>
<description>
The actual address the server will bind to. If this optional address is
set, the RPC and webapp servers will bind to this address and the port specified in
yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively.
This is most useful for making the service listen to all interfaces by setting to
0.0.0.0.
</description>
<name>yarn.timeline-service.bind-host</name>
<value></value>
</property>
<property>
<description>
Defines the max number of applications could be fetched using REST API or
application history protocol and shown in timeline server web ui.
</description>
<name>yarn.timeline-service.generic-application-history.max-applications</name>
<value>10000</value>
</property>
<property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
<description>Enable age off of timeline store data.</description>
<name>yarn.timeline-service.ttl-enable</name>
<value>true</value>
</property>
<property>
<description>Time to live for timeline store data in milliseconds.</description>
<name>yarn.timeline-service.ttl-ms</name>
<value>604800000</value>
</property>
<property>
<description>Store file name for leveldb timeline store.</description>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
<value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
<property>
<description>Length of time to wait between deletion cycles of leveldb timeline store in milliseconds.</description>
<name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
<value>300000</value>
</property>
<property>
<description>Size of read cache for uncompressed blocks for leveldb timeline store in bytes.</description>
<name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
<value>104857600</value>
</property>
<property>
<description>Size of cache for recently read entity start times for leveldb timeline store in number of entities.</description>
<name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
<value>10000</value>
</property>
<property>
<description>Size of cache for recently written entity start times for leveldb timeline store in number of entities.</description>
<name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
<value>10000</value>
</property>
<property>
<description>Handler thread count to serve the client RPC requests.</description>
<name>yarn.timeline-service.handler-thread-count</name>
<value>10</value>
</property>
<property>
<name>yarn.timeline-service.http-authentication.type</name>
<value>simple</value>
<description>
Defines authentication used for the timeline server HTTP endpoint.
Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
</description>
</property>
<property>
<name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
<value>true</value>
<description>
Indicates if anonymous requests are allowed by the timeline server when using
'simple' authentication.
</description>
</property>
<property>
<description>The Kerberos principal for the timeline server.</description>
<name>yarn.timeline-service.principal</name>
<value></value>
</property>
<property>
<description>The Kerberos keytab for the timeline server.</description>
<name>yarn.timeline-service.keytab</name>
<value>/etc/krb5.keytab</value>
</property>
<property>
<description>Comma separated list of UIs that will be hosted</description>
<name>yarn.timeline-service.ui-names</name>
<value></value>
</property>
<property>
<description>
Default maximum number of retries for timeline service client
and value -1 means no limit.
</description>
<name>yarn.timeline-service.client.max-retries</name>
<value>30</value>
</property>
<property>
<description>Client policy for whether timeline operations are non-fatal.
Should the failure to obtain a delegation token be considered an application
failure (option = false), or should the client attempt to continue to
publish information without it (option=true)</description>
<name>yarn.timeline-service.client.best-effort</name>
<value>false</value>
</property>
<property>
<description>
Default retry time interval for timeline servive client.
</description>
<name>yarn.timeline-service.client.retry-interval-ms</name>
<value>1000</value>
</property>
<property>
<description>
The time period for which timeline v2 client will wait for draining
leftover entities after stop.
</description>
<name>yarn.timeline-service.client.drain-entities.timeout.ms</name>
<value>2000</value>
</property>
<property>
<description>Enable timeline server to recover state after starting. If
true, then yarn.timeline-service.state-store-class must be specified.
</description>
<name>yarn.timeline-service.recovery.enabled</name>
<value>false</value>
</property>
<property>
<description>Store class name for timeline state store.</description>
<name>yarn.timeline-service.state-store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
</property>
<property>
<description>Store file name for leveldb state store.</description>
<name>yarn.timeline-service.leveldb-state-store.path</name>
<value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
<!-- Timeline Service v1.5 Configuration -->
<property>
<name>yarn.timeline-service.entity-group-fs-store.cache-store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore</value>
<description>Caching storage timeline server v1.5 is using. </description>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
<value>/tmp/entity-file-history/active</value>
<description>HDFS path to store active application’s timeline data</description>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
<value>/tmp/entity-file-history/done/</value>
<description>HDFS path to store done application’s timeline data</description>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
<value></value>
<description>
Plugins that can translate a timeline entity read request into
a list of timeline entity group ids, separated by commas.
</description>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath</name>
<value></value>
<description>
Classpath for all plugins defined in
yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes.
</description>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
<description>Summary storage for ATS v1.5</description>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
<description>
Scan interval for ATS v1.5 entity group file system storage reader.This
value controls how frequent the reader will scan the HDFS active directory
for application status.
</description>
<value>60</value>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
<description>
Scan interval for ATS v1.5 entity group file system storage cleaner.This
value controls how frequent the reader will scan the HDFS done directory
for stale application data.
</description>
<value>3600</value>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
<description>
How long the ATS v1.5 entity group file system storage will keep an
application's data in the done directory.
</description>
<value>604800</value>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size</name>
<description>
Read cache size for the leveldb cache storage in ATS v1.5 plugin storage.
</description>
<value>10485760</value>
</property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.app-cache-size</name>
<description>
Size of the reader cache for ATS v1.5 reader. This value controls how many
entity groups the ATS v1.5 server should cache. If the number of active
read entity groups is greater than the number of caches items, some reads
may return empty data. This value must be greater than 0.
</description>
<value>10</value>
</property>
<property>
<name>yarn.timeline-service.client.fd-flush-interval-secs</name>
<description>
Flush interval for ATS v1.5 writer. This value controls how frequent
the writer will flush the HDFS FSStream for the entity/domain.
</description>
<value>10</value>
</property>
<property>
<name>yarn.timeline-service.client.fd-clean-interval-secs</name>
<description>
Scan interval for ATS v1.5 writer. This value controls how frequent
the writer will scan the HDFS FSStream for the entity/domain.
If the FSStream is stale for a long time, this FSStream will be close.
</description>
<value>60</value>
</property>
<property>
<name>yarn.timeline-service.client.fd-retain-secs</name>
<description>
How long the ATS v1.5 writer will keep an FSStream open.
If this fsstream does not write anything for this configured time,
it will be close.
</description>
<value>300</value>
</property>
<!-- Timeline Service v2 Configuration -->
<property>
<name>yarn.timeline-service.writer.class</name>
<description>
Storage implementation ATS v2 will use for the TimelineWriter service.
</description>
<value>org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl</value>
</property>
<property>
<name>yarn.timeline-service.reader.class</name>
<description>
Storage implementation ATS v2 will use for the TimelineReader service.
</description>
<value>org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl</value>
</property>
<property>
<name>yarn.timeline-service.client.internal-timers-ttl-secs</name>
<description>
How long the internal Timer Tasks can be alive in writer. If there is no
write operation for this configured time, the internal timer tasks will
be close.
</description>
<value>420</value>
</property>
<property>
<description>The setting that controls how often the timeline collector
flushes the timeline writer.</description>
<name>yarn.timeline-service.writer.flush-interval-seconds</name>
<value>60</value>
</property>
<property>
<description>Time period till which the application collector will be alive
in NM, after the application master container finishes.</description>
<name>yarn.timeline-service.app-collector.linger-period.ms</name>
<value>60000</value>
</property>
<property>
<description>Time line V2 client tries to merge these many number of
async entities (if available) and then call the REST ATS V2 API to submit.
</description>
<name>yarn.timeline-service.timeline-client.number-of-async-entities-to-merge</name>
<value>10</value>
</property>
<property>
<description>
The setting that controls how long the final value
of a metric of a completed app is retained before merging into
the flow sum. Up to this time after an application is completed
out-of-order values that arrive can be recognized and discarded at the
cost of increased storage.
</description>
<name>yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds
</name>
<value>259200000</value>
</property>
<property>
<description>
The default hdfs location for flowrun coprocessor jar.
</description>
<name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location
</name>
<value>/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar</value>
</property>
<property>
<description>
The value of this parameter sets the prefix for all tables that are part of
timeline service in the hbase storage schema. It can be set to "dev."
or "staging." if it is to be used for development or staging instances.
This way the data in production tables stays in a separate set of tables
prefixed by "prod.".
</description>
<name>yarn.timeline-service.hbase-schema.prefix</name>
<value>prod.</value>
</property>
<property>
<description> Optional URL to an hbase-site.xml configuration file to be
used to connect to the timeline-service hbase cluster. If empty or not
specified, then the HBase configuration will be loaded from the classpath.
When specified the values in the specified configuration file will override
those from the ones that are present on the classpath.
</description>
<name>yarn.timeline-service.hbase.configuration.file
</name>
<value></value>
</property>
<!-- Shared Cache Configuration -->
<property>
<description>Whether the shared cache is enabled</description>
<name>yarn.sharedcache.enabled</name>
<value>false</value>
</property>
<property>
<description>The root directory for the shared cache</description>
<name>yarn.sharedcache.root-dir</name>
<value>/sharedcache</value>
</property>
<property>
<description>The level of nested directories before getting to the checksum
directories. It must be non-negative.</description>
<name>yarn.sharedcache.nested-level</name>
<value>3</value>
</property>
<property>
<description>The implementation to be used for the SCM store</description>
<name>yarn.sharedcache.store.class</name>
<value>org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore</value>
</property>
<property>
<description>The implementation to be used for the SCM app-checker</description>
<name>yarn.sharedcache.app-checker.class</name>
<value>org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker</value>
</property>
<property>
<description>A resource in the in-memory store is considered stale
if the time since the last reference exceeds the staleness period.
This value is specified in minutes.</description>
<name>yarn.sharedcache.store.in-memory.staleness-period-mins</name>
<value>10080</value>
</property>
<property>
<description>Initial delay before the in-memory store runs its first check
to remove dead initial applications. Specified in minutes.</description>
<name>yarn.sharedcache.store.in-memory.initial-delay-mins</name>
<value>10</value>
</property>
<property>
<description>The frequency at which the in-memory store checks to remove
dead initial applications. Specified in minutes.</description>
<name>yarn.sharedcache.store.in-memory.check-period-mins</name>
<value>720</value>
</property>
<property>
<description>The address of the admin interface in the SCM (shared cache manager)</description>
<name>yarn.sharedcache.admin.address</name>
<value>0.0.0.0:8047</value>
</property>
<property>
<description>The number of threads used to handle SCM admin interface (1 by default)</description>
<name>yarn.sharedcache.admin.thread-count</name>
<value>1</value>
</property>
<property>
<description>The address of the web application in the SCM (shared cache manager)</description>
<name>yarn.sharedcache.webapp.address</name>
<value>0.0.0.0:8788</value>
</property>
<property>
<description>The frequency at which a cleaner task runs.
Specified in minutes.</description>
<name>yarn.sharedcache.cleaner.period-mins</name>
<value>1440</value>
</property>
<property>
<description>Initial delay before the first cleaner task is scheduled.
Specified in minutes.</description>
<name>yarn.sharedcache.cleaner.initial-delay-mins</name>
<value>10</value>
</property>
<property>
<description>The time to sleep between processing each shared cache
resource. Specified in milliseconds.</description>
<name>yarn.sharedcache.cleaner.resource-sleep-ms</name>
<value>0</value>
</property>
<property>
<description>The address of the node manager interface in the SCM
(shared cache manager)</description>
<name>yarn.sharedcache.uploader.server.address</name>
<value>0.0.0.0:8046</value>
</property>
<property>
<description>The number of threads used to handle shared cache manager
requests from the node manager (50 by default)</description>
<name>yarn.sharedcache.uploader.server.thread-count</name>
<value>50</value>
</property>
<property>
<description>The address of the client interface in the SCM
(shared cache manager)</description>
<name>yarn.sharedcache.client-server.address</name>
<value>0.0.0.0:8045</value>
</property>
<property>
<description>The number of threads used to handle shared cache manager
requests from clients (50 by default)</description>
<name>yarn.sharedcache.client-server.thread-count</name>
<value>50</value>
</property>
<property>
<description>The algorithm used to compute checksums of files (SHA-256 by
default)</description>
<name>yarn.sharedcache.checksum.algo.impl</name>
<value>org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl</value>
</property>
<property>
<description>The replication factor for the node manager uploader for the
shared cache (10 by default)</description>
<name>yarn.sharedcache.nm.uploader.replication.factor</name>
<value>10</value>
</property>
<property>
<description>The number of threads used to upload files from a node manager
instance (20 by default)</description>
<name>yarn.sharedcache.nm.uploader.thread-count</name>
<value>20</value>
</property>
<property>
<description>
ACL protocol for use in the Timeline server.
</description>
<name>security.applicationhistory.protocol.acl</name>
<value></value>
</property>
<!-- Minicluster Configuration (for testing only!) -->
<property>
<description>
Set to true for MiniYARNCluster unit tests
</description>
<name>yarn.is.minicluster</name>
<value>false</value>
</property>
<property>
<description>
Set for MiniYARNCluster unit tests to control resource monitoring
</description>
<name>yarn.minicluster.control-resource-monitoring</name>
<value>false</value>
</property>
<property>
<description>
Set to false in order to allow MiniYARNCluster to run tests without
port conflicts.
</description>
<name>yarn.minicluster.fixed.ports</name>
<value>false</value>
</property>
<property>
<description>
Set to false in order to allow the NodeManager in MiniYARNCluster to
use RPC to talk to the RM.
</description>
<name>yarn.minicluster.use-rpc</name>
<value>false</value>
</property>
<property>
<description>
As yarn.nodemanager.resource.memory-mb property but for the NodeManager
in a MiniYARNCluster.
</description>
<name>yarn.minicluster.yarn.nodemanager.resource.memory-mb</name>
<value>4096</value>
</property>
<!-- Node Labels Configuration -->
<property>
<description>
Enable node labels feature
</description>
<name>yarn.node-labels.enabled</name>
<value>false</value>
</property>
<property>
<description>
URI for NodeLabelManager. The default value is
/tmp/hadoop-yarn-${user}/node-labels/ in the local filesystem.
</description>
<name>yarn.node-labels.fs-store.root-dir</name>
<value></value>
</property>
<property>
<description>
Set configuration type for node labels. Administrators can specify
"centralized", "delegated-centralized" or "distributed".
</description>
<name>yarn.node-labels.configuration-type</name>
<value>centralized</value>
</property>
<!-- Distributed Node Labels Configuration -->
<property>
<description>
When "yarn.node-labels.configuration-type" is configured with "distributed"
in RM, Administrators can configure in NM the provider for the
node labels by configuring this parameter. Administrators can
configure "config", "script" or the class name of the provider. Configured
class needs to extend
org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider.
If "config" is configured, then "ConfigurationNodeLabelsProvider" and if
"script" is configured, then "ScriptNodeLabelsProvider" will be used.
</description>
<name>yarn.nodemanager.node-labels.provider</name>
</property>
<property>
<description>
When "yarn.nodemanager.node-labels.provider" is configured with "config",
"Script" or the configured class extends AbstractNodeLabelsProvider, then
periodically node labels are retrieved from the node labels provider. This
configuration is to define the interval period.
If -1 is configured then node labels are retrieved from provider only
during initialization. Defaults to 10 mins.
</description>
<name>yarn.nodemanager.node-labels.provider.fetch-interval-ms</name>
<value>600000</value>
</property>
<property>
<description>
Interval at which NM syncs its node labels with RM. NM will send its loaded
labels every x intervals configured, along with heartbeat to RM.
</description>
<name>yarn.nodemanager.node-labels.resync-interval-ms</name>
<value>120000</value>
</property>
<property>
<description>
When "yarn.nodemanager.node-labels.provider" is configured with "config"
then ConfigurationNodeLabelsProvider fetches the partition label from this
parameter.
</description>
<name>yarn.nodemanager.node-labels.provider.configured-node-partition</name>
</property>
<property>
<description>
When "yarn.nodemanager.node-labels.provider" is configured with "Script"
then this configuration provides the timeout period after which it will
interrupt the script which queries the Node labels. Defaults to 20 mins.
</description>
<name>yarn.nodemanager.node-labels.provider.fetch-timeout-ms</name>
<value>1200000</value>
</property>
<!-- Delegated-centralized Node Labels Configuration -->
<property>
<description>
When node labels "yarn.node-labels.configuration-type" is
of type "delegated-centralized", administrators should configure
the class for fetching node labels by ResourceManager. Configured
class needs to extend
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.
RMNodeLabelsMappingProvider.
</description>
<name>yarn.resourcemanager.node-labels.provider</name>
<value></value>
</property>
<property>
<description>
When "yarn.node-labels.configuration-type" is configured with
"delegated-centralized", then periodically node labels are retrieved
from the node labels provider. This configuration is to define the
interval. If -1 is configured then node labels are retrieved from
provider only once for each node after it registers. Defaults to 30 mins.
</description>
<name>yarn.resourcemanager.node-labels.provider.fetch-interval-ms</name>
<value>1800000</value>
</property>
<property>
<description>
Timeout in seconds for YARN node graceful decommission.
This is the maximal time to wait for running containers and applications to complete
before transition a DECOMMISSIONING node into DECOMMISSIONED.
</description>
<name>yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs</name>
<value>3600</value>
</property>
<property>
<description>
Timeout in seconds of DecommissioningNodesWatcher internal polling.
</description>
<name>yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs</name>
<value>20</value>
</property>
<property>
<description>The Node Label script to run. Script output Line starting with
"NODE_PARTITION:" will be considered as Node Label Partition. In case of
multiple lines have this pattern, then last one will be considered
</description>
<name>yarn.nodemanager.node-labels.provider.script.path</name>
</property>
<property>
<description>The arguments to pass to the Node label script.</description>
<name>yarn.nodemanager.node-labels.provider.script.opts</name>
</property>
<!-- Federation Configuration -->
<property>
<description>
Flag to indicate whether the RM is participating in Federation or not.
</description>
<name>yarn.federation.enabled</name>
<value>false</value>
</property>
<property>
<description>
Machine list file to be loaded by the FederationSubCluster Resolver
</description>
<name>yarn.federation.machine-list</name>
</property>
<property>
<description>
Class name for SubClusterResolver
</description>
<name>yarn.federation.subcluster-resolver.class</name>
<value>org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl</value>
</property>
<property>
<description>
Store class name for federation state store
</description>
<name>yarn.federation.state-store.class</name>
<value>org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore</value>
</property>
<property>
<description>
The time in seconds after which the federation state store local cache
will be refreshed periodically
</description>
<name>yarn.federation.cache-ttl.secs</name>
<value>300</value>
</property>
<property>
<description>The registry base directory for federation.</description>
<name>yarn.federation.registry.base-dir</name>
<value>yarnfederation/</value>
</property>
<!-- Other Configuration -->
<property>
<description>The registry implementation to use.</description>
<name>yarn.registry.class</name>
<value>org.apache.hadoop.registry.client.impl.FSRegistryOperationsService</value>
</property>
<property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
</description>
<name>yarn.client.application-client-protocol.poll-interval-ms</name>
<value>200</value>
</property>
<property>
<description>
The duration (in ms) the YARN client waits for an expected state change
to occur. -1 means unlimited wait time.
</description>
<name>yarn.client.application-client-protocol.poll-timeout-ms</name>
<value>-1</value>
</property>
<property>
<description>RSS usage of a process computed via
/proc/pid/stat is not very accurate as it includes shared pages of a
process. /proc/pid/smaps provides useful information like
Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used
for computing more accurate RSS. When this flag is enabled, RSS is computed
as Min(Shared_Dirty, Pss) + Private_Clean + Private_Dirty. It excludes
read-only shared mappings in RSS computation.
</description>
<name>yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled</name>
<value>false</value>
</property>
<property>
<description>
URL for log aggregation server
</description>
<name>yarn.log.server.url</name>
<value></value>
</property>
<property>
<description>
URL for log aggregation server web service
</description>
<name>yarn.log.server.web-service.url</name>
<value></value>
</property>
<property>
<description>
RM Application Tracking URL
</description>
<name>yarn.tracking.url.generator</name>
<value></value>
</property>
<property>
<description>
Class to be used for YarnAuthorizationProvider
</description>
<name>yarn.authorization-provider</name>
<value></value>
</property>
<property>
<description>Defines how often NMs wake up to upload log files.
The default value is -1. By default, the logs will be uploaded when
the application is finished. By setting this configure, logs can be uploaded
periodically when the application is running. The minimum rolling-interval-seconds
can be set is 3600.
</description>
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
<value>-1</value>
</property>
<property>
<description>Define how many aggregated log files per application per NM
we can have in remote file system. By default, the total number of
aggregated log files per application per NM is 30.
</description>
<name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
<value>30</value>
</property>
<property>
<description>
Enable/disable intermediate-data encryption at YARN level. For now,
this only is used by the FileSystemRMStateStore to setup right
file-system security attributes.
</description>
<name>yarn.intermediate-data-encryption.enable</name>
<value>false</value>
</property>
<property>
<description>Flag to enable cross-origin (CORS) support in the NM. This flag
requires the CORS filter initializer to be added to the filter initializers
list in core-site.xml.</description>
<name>yarn.nodemanager.webapp.cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>
Defines maximum application priority in a cluster.
If an application is submitted with a priority higher than this value, it will be
reset to this maximum value.
</description>
<name>yarn.cluster.max-application-priority</name>
<value>0</value>
</property>
<property>
<description>
The default log aggregation policy class. Applications can
override it via LogAggregationContext. This configuration can provide
some cluster-side default behavior so that if the application doesn't
specify any policy via LogAggregationContext administrators of the cluster
can adjust the policy globally.
</description>
<name>yarn.nodemanager.log-aggregation.policy.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy</value>
</property>
<property>
<description>
The default parameters for the log aggregation policy. Applications can
override it via LogAggregationContext. This configuration can provide
some cluster-side default behavior so that if the application doesn't
specify any policy via LogAggregationContext administrators of the cluster
can adjust the policy globally.
</description>
<name>yarn.nodemanager.log-aggregation.policy.parameters</name>
<value></value>
</property>
<property>
<description>
Enable/Disable AMRMProxyService in the node manager. This service is used to
intercept calls from the application masters to the resource manager.
</description>
<name>yarn.nodemanager.amrmproxy.enabled</name>
<value>false</value>
</property>
<property>
<description>
The address of the AMRMProxyService listener.
</description>
<name>yarn.nodemanager.amrmproxy.address</name>
<value>0.0.0.0:8049</value>
</property>
<property>
<description>
The number of threads used to handle requests by the AMRMProxyService.
</description>
<name>yarn.nodemanager.amrmproxy.client.thread-count</name>
<value>25</value>
</property>
<property>
<description>
The comma separated list of class names that implement the
RequestInterceptor interface. This is used by the AMRMProxyService to create
the request processing pipeline for applications.
</description>
<name>yarn.nodemanager.amrmproxy.interceptor-class.pipeline</name>
<value>org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor</value>
</property>
<property>
<description>
Whether AMRMProxy HA is enabled.
</description>
<name>yarn.nodemanager.amrmproxy.ha.enable</name>
<value>false</value>
</property>
<property>
<description>
Setting that controls whether distributed scheduling is enabled.
</description>
<name>yarn.nodemanager.distributed-scheduling.enabled</name>
<value>false</value>
</property>
<property>
<description>
Setting that controls whether opportunistic container allocation
is enabled.
</description>
<name>yarn.resourcemanager.opportunistic-container-allocation.enabled</name>
<value>false</value>
</property>
<property>
<description>
Number of nodes to be used by the Opportunistic Container Allocator for
dispatching containers during container allocation.
</description>
<name>yarn.resourcemanager.opportunistic-container-allocation.nodes-used</name>
<value>10</value>
</property>
<property>
<description>
Frequency for computing least loaded NMs.
</description>
<name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
<value>1000</value>
</property>
<property>
<description>
Comparator for determining node load for Distributed Scheduling.
</description>
<name>yarn.resourcemanager.nm-container-queuing.load-comparator</name>
<value>QUEUE_LENGTH</value>
</property>
<property>
<description>
Value of standard deviation used for calculation of queue limit thresholds.
</description>
<name>yarn.resourcemanager.nm-container-queuing.queue-limit-stdev</name>
<value>1.0f</value>
</property>
<property>
<description>
Min length of container queue at NodeManager.
</description>
<name>yarn.resourcemanager.nm-container-queuing.min-queue-length</name>
<value>5</value>
</property>
<property>
<description>
Max length of container queue at NodeManager.
</description>
<name>yarn.resourcemanager.nm-container-queuing.max-queue-length</name>
<value>15</value>
</property>
<property>
<description>
Min queue wait time for a container at a NodeManager.
</description>
<name>yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms</name>
<value>10</value>
</property>
<property>
<description>
Max queue wait time for a container queue at a NodeManager.
</description>
<name>yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms</name>
<value>100</value>
</property>
<property>
<description>
Use container pause as the preemption policy over kill in the container
queue at a NodeManager.
</description>
<name>yarn.nodemanager.opportunistic-containers-use-pause-for-preemption</name>
<value>false</value>
</property>
<property>
<description>
Error filename pattern, to identify the file in the container's
Log directory which contain the container's error log. As error file
redirection is done by client/AM and yarn will not be aware of the error
file name. YARN uses this pattern to identify the error file and tail
the error log as diagnostics when the container execution returns non zero
value. Filename patterns are case sensitive and should match the
specifications of FileSystem.globStatus(Path) api. If multiple filenames
matches the pattern, first file matching the pattern will be picked.
</description>
<name>yarn.nodemanager.container.stderr.pattern</name>
<value>{*stderr*,*STDERR*}</value>
</property>
<property>
<description>
Size of the container error file which needs to be tailed, in bytes.
</description>
<name>yarn.nodemanager.container.stderr.tail.bytes </name>
<value>4096</value>
</property>
<property>
<description>
Choose different implementation of node label's storage
</description>
<name>yarn.node-labels.fs-store.impl.class</name>
<value>org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore</value>
</property>
<property>
<description>
Enable the CSRF filter for the RM web app
</description>
<name>yarn.resourcemanager.webapp.rest-csrf.enabled</name>
<value>false</value>
</property>
<property>
<description>
Optional parameter that indicates the custom header name to use for CSRF
protection.
</description>
<name>yarn.resourcemanager.webapp.rest-csrf.custom-header</name>
<value>X-XSRF-Header</value>
</property>
<property>
<description>
Optional parameter that indicates the list of HTTP methods that do not
require CSRF protection
</description>
<name>yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD</value>
</property>
<property>
<description>
Enable the CSRF filter for the NM web app
</description>
<name>yarn.nodemanager.webapp.rest-csrf.enabled</name>
<value>false</value>
</property>
<property>
<description>
Optional parameter that indicates the custom header name to use for CSRF
protection.
</description>
<name>yarn.nodemanager.webapp.rest-csrf.custom-header</name>
<value>X-XSRF-Header</value>
</property>
<property>
<description>
Optional parameter that indicates the list of HTTP methods that do not
require CSRF protection
</description>
<name>yarn.nodemanager.webapp.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD</value>
</property>
<property>
<description>
The name of disk validator.
</description>
<name>yarn.nodemanager.disk-validator</name>
<value>basic</value>
</property>
<property>
<description>
Enable the CSRF filter for the timeline service web app
</description>
<name>yarn.timeline-service.webapp.rest-csrf.enabled</name>
<value>false</value>
</property>
<property>
<description>
Optional parameter that indicates the custom header name to use for CSRF
protection.
</description>
<name>yarn.timeline-service.webapp.rest-csrf.custom-header</name>
<value>X-XSRF-Header</value>
</property>
<property>
<description>
Optional parameter that indicates the list of HTTP methods that do not
require CSRF protection
</description>
<name>yarn.timeline-service.webapp.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD</value>
</property>
<property>
<description>
Enable the XFS filter for YARN
</description>
<name>yarn.webapp.xfs-filter.enabled</name>
<value>true</value>
</property>
<property>
<description>
Property specifying the xframe options value.
</description>
<name>yarn.resourcemanager.webapp.xfs-filter.xframe-options</name>
<value>SAMEORIGIN</value>
</property>
<property>
<description>
Property specifying the xframe options value.
</description>
<name>yarn.nodemanager.webapp.xfs-filter.xframe-options</name>
<value>SAMEORIGIN</value>
</property>
<property>
<description>
Property specifying the xframe options value.
</description>
<name>yarn.timeline-service.webapp.xfs-filter.xframe-options</name>
<value>SAMEORIGIN</value>
</property>
<property>
<description>
The least amount of time(msec.) an inactive (decommissioned or shutdown) node can
stay in the nodes list of the resourcemanager after being declared untracked.
A node is marked untracked if and only if it is absent from both include and
exclude nodemanager lists on the RM. All inactive nodes are checked twice per
timeout interval or every 10 minutes, whichever is lesser, and marked appropriately.
The same is done when refreshNodes command (graceful or otherwise) is invoked.
</description>
<name>yarn.resourcemanager.node-removal-untracked.timeout-ms</name>
<value>60000</value>
</property>
<property>
<description>
The RMAppLifetimeMonitor Service uses this value as monitor interval
</description>
<name>yarn.resourcemanager.application-timeouts.monitor.interval-ms</name>
<value>3000</value>
</property>
<property>
<description>
Defines the limit of the diagnostics message of an application
attempt, in kilo characters (character count * 1024).
When using ZooKeeper to store application state behavior, it's
important to limit the size of the diagnostic messages to
prevent YARN from overwhelming ZooKeeper. In cases where
yarn.resourcemanager.state-store.max-completed-applications is set to
a large number, it may be desirable to reduce the value of this property
to limit the total data stored.
</description>
<name>yarn.app.attempt.diagnostics.limit.kc</name>
<value>64</value>
</property>
<property>
<description>
Flag to enable cross-origin (CORS) support for timeline service v1.x or
Timeline Reader in timeline service v2. For timeline service v2, also add
org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the
configuration hadoop.http.filter.initializers in core-site.xml.
</description>
<name>yarn.timeline-service.http-cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>
Flag to enable cross-origin (CORS) support for timeline service v1.x or
Timeline Reader in timeline service v2. For timeline service v2, also add
org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the
configuration hadoop.http.filter.initializers in core-site.xml.
</description>
<name>yarn.timeline-service.http-cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>
The comma separated list of class names that implement the
RequestInterceptor interface. This is used by the RouterClientRMService
to create the request processing pipeline for users.
</description>
<name>yarn.router.clientrm.interceptor-class.pipeline</name>
<value>org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor</value>
</property>
<property>
<description>
Size of LRU cache for Router ClientRM Service and RMAdmin Service.
</description>
<name>yarn.router.pipeline.cache-max-size</name>
<value>25</value>
</property>
<property>
<description>
The comma separated list of class names that implement the
RequestInterceptor interface. This is used by the RouterRMAdminService
to create the request processing pipeline for users.
</description>
<name>yarn.router.rmadmin.interceptor-class.pipeline</name>
<value>org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor</value>
</property>
<property>
<description>
The actual address the server will bind to. If this optional address is
set, the RPC and webapp servers will bind to this address and the port specified in
yarn.router.address and yarn.router.webapp.address, respectively. This is
most useful for making Router listen to all interfaces by setting to 0.0.0.0.
</description>
<name>yarn.router.bind-host</name>
<value></value>
</property>
<property>
<description>
Comma-separated list of PlacementRules to determine how applications
submitted by certain users get mapped to certain queues. Default is
user-group, which corresponds to UserGroupMappingPlacementRule.
</description>
<name>yarn.scheduler.queue-placement-rules</name>
<value>user-group</value>
</property>
<property>
<description>
The comma separated list of class names that implement the
RequestInterceptor interface. This is used by the RouterWebServices
to create the request processing pipeline for users.
</description>
<name>yarn.router.webapp.interceptor-class.pipeline</name>
<value>org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST</value>
</property>
<property>
<description>
The http address of the Router web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name>yarn.router.webapp.address</name>
<value>0.0.0.0:8089</value>
</property>
<property>
<description>
The https address of the Router web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name> yarn.router.webapp.https.address</name>
<value>0.0.0.0:8091</value>
</property>
<property>
<description>
It is TimelineClient 1.5 configuration whether to store active
application’s timeline data with in user directory i.e
${yarn.timeline-service.entity-group-fs-store.active-dir}/${user.name}
</description>
<name>yarn.timeline-service.entity-group-fs-store.with-user-dir</name>
<value>false</value>
</property>
<!-- resource types configuration -->
<property>
<name>yarn.resource-types</name>
<value></value>
<description>
The resource types to be used for scheduling. Use resource-types.xml
to specify details about the individual resource types.
</description>
</property>
<property>
<name>yarn.webapp.filter-entity-list-by-user</name>
<value>false</value>
<description>
Flag to enable display of applications per user as an admin
configuration.
</description>
</property>
<property>
<description>
The type of configuration store to use for scheduler configurations.
Default is "file", which uses file based capacity-scheduler.xml to
retrieve and change scheduler configuration. To enable API based
scheduler configuration, use either "memory" (in memory storage, no
persistence across restarts), "leveldb" (leveldb based storage), or
"zk" (zookeeper based storage). API based configuration is only useful
when using a scheduler which supports mutable configuration. Currently
only capacity scheduler supports this.
</description>
<name>yarn.scheduler.configuration.store.class</name>
<value>file</value>
</property>
<property>
<description>
The class to use for configuration mutation ACL policy if using a mutable
configuration provider. Controls whether a mutation request is allowed.
The DefaultConfigurationMutationACLPolicy checks if the requestor is a
YARN admin.
</description>
<name>yarn.scheduler.configuration.mutation.acl-policy.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
</property>
<property>
<description>
The storage path for LevelDB implementation of configuration store,
when yarn.scheduler.configuration.store.class is configured to be
"leveldb".
</description>
<name>yarn.scheduler.configuration.leveldb-store.path</name>
<value>${hadoop.tmp.dir}/yarn/system/confstore</value>
</property>
<property>
<description>
The compaction interval for LevelDB configuration store in secs,
when yarn.scheduler.configuration.store.class is configured to be
"leveldb". Default is one day.
</description>
<name>yarn.scheduler.configuration.leveldb-store.compaction-interval-secs</name>
<value>86400</value>
</property>
<property>
<description>
The max number of configuration change log entries kept in config
store, when yarn.scheduler.configuration.store.class is configured to be
"leveldb" or "zk". Default is 1000 for either.
</description>
<name>yarn.scheduler.configuration.store.max-logs</name>
<value>1000</value>
</property>
<property>
<description>
ZK root node path for configuration store when using zookeeper-based
configuration store.
</description>
<name>yarn.scheduler.configuration.zk-store.parent-path</name>
<value>/confstore</value>
</property>
<property>
<description>
Provides an option for client to load supported resource types from RM
instead of depending on local resource-types.xml file.
</description>
<name>yarn.client.load.resource-types.from-server</name>
<value>false</value>
</property>
<property>
<description>
When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
YARN NodeManager needs to run GPU discovery binary (now only support
nvidia-smi) to get GPU-related information.
When value is empty (default), YARN NodeManager will try to locate
discovery executable itself.
An example of the config value is: /usr/local/bin/nvidia-smi
</description>
<name>yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables</name>
<value></value>
</property>
<property>
<description>
Enable additional discovery/isolation of resources on the NodeManager,
split by comma. By default, this is empty.
Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}.
</description>
<name>yarn.nodemanager.resource-plugins</name>
<value></value>
</property>
<property>
<description>
Specify GPU devices which can be managed by YARN NodeManager, split by comma
Number of GPU devices will be reported to RM to make scheduling decisions.
Set to auto (default) let YARN automatically discover GPU resource from
system.
Manually specify GPU devices if auto detect GPU device failed or admin
only want subset of GPU devices managed by YARN. GPU device is identified
by their minor device number and index. A common approach to get minor
device number of GPUs is using "nvidia-smi -q" and search "Minor Number"
output.
When manual specify minor numbers, admin needs to include indice of GPUs
as well, format is index:minor_number[,index:minor_number...]. An example
of manual specification is "0:0,1:1,2:2,3:4" to allow YARN NodeManager to
manage GPU devices with indice 0/1/2/3 and minor number 0/1/2/4.
numbers .
</description>
<name>yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices</name>
<value>auto</value>
</property>
<property>
<description>
Specify docker command plugin for GPU. By default uses Nvidia docker V1.
</description>
<name>yarn.nodemanager.resource-plugins.gpu.docker-plugin</name>
<value>nvidia-docker-v1</value>
</property>
<property>
<description>
Specify end point of nvidia-docker-plugin.
Please find documentation: https://github.com/NVIDIA/nvidia-docker/wiki
For more details.
</description>
<name>yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint</name>
<value>http://localhost:3476/v1.0/docker/cli</value>
</property>
<property>
<description>
Specify one vendor plugin to handle FPGA devices discovery/IP download/configure.
Only IntelFpgaOpenclPlugin is supported by default.
We only allow one NM configured with one vendor FPGA plugin now since the end user can put the same
vendor's cards in one host. And this also simplify our design.
</description>
<name>yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin</value>
</property>
<property>
<description>
When yarn.nodemanager.resource.fpga.allowed-fpga-devices=auto specified,
YARN NodeManager needs to run FPGA discovery binary (now only support
IntelFpgaOpenclPlugin) to get FPGA information.
When value is empty (default), YARN NodeManager will try to locate
discovery executable from vendor plugin's preference
</description>
<name>yarn.nodemanager.resource-plugins.fpga.path-to-discovery-executables</name>
<value></value>
</property>
<property>
<description>
Specify FPGA devices which can be managed by YARN NodeManager, split by comma
Number of FPGA devices will be reported to RM to make scheduling decisions.
Set to auto (default) let YARN automatically discover FPGA resource from
system.
Manually specify FPGA devices if admin only want subset of FPGA devices managed by YARN.
At present, since we can only configure one major number in c-e.cfg, FPGA device is
identified by their minor device number. A common approach to get minor
device number of FPGA is using "aocl diagnose" and check uevent with device name.
</description>
<name>yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices</name>
<value>0,1</value>
</property>
<property>
<description>The http address of the timeline reader web application.</description>
<name>yarn.timeline-service.reader.webapp.address</name>
<value>${yarn.timeline-service.webapp.address}</value>
</property>
<property>
<description>The https address of the timeline reader web application.</description>
<name>yarn.timeline-service.reader.webapp.https.address</name>
<value>${yarn.timeline-service.webapp.https.address}</value>
</property>
<property>
<description>
The actual address timeline reader will bind to. If this optional address is
set, the reader server will bind to this address and the port specified in
yarn.timeline-service.reader.webapp.address.
This is most useful for making the service listen to all interfaces by setting to
0.0.0.0.
</description>
<name>yarn.timeline-service.reader.bind-host</name>
<value></value>
</property>
<property>
<description>
Whether to enable the NUMA awareness for containers in Node Manager.
</description>
<name>yarn.nodemanager.numa-awareness.enabled</name>
<value>false</value>
</property>
<property>
<description>
Whether to read the NUMA topology from the system or from the
configurations. If the value is true then NM reads the NUMA topology from
system using the command 'numactl --hardware'. If the value is false then NM
reads the topology from the configurations
'yarn.nodemanager.numa-awareness.node-ids'(for node id's),
'yarn.nodemanager.numa-awareness.<NODE_ID>.memory'(for each node memory),
'yarn.nodemanager.numa-awareness.<NODE_ID>.cpus'(for each node cpus).
</description>
<name>yarn.nodemanager.numa-awareness.read-topology</name>
<value>false</value>
</property>
<property>
<description>
NUMA node id's in the form of comma separated list. Memory and No of CPUs
will be read using the properties
'yarn.nodemanager.numa-awareness.<NODE_ID>.memory' and
'yarn.nodemanager.numa-awareness.<NODE_ID>.cpus' for each id specified
in this value. This property value will be read only when
'yarn.nodemanager.numa-awareness.read-topology=false'.
For example, if yarn.nodemanager.numa-awareness.node-ids=0,1
then need to specify memory and cpus for node id's '0' and '1' like below,
yarn.nodemanager.numa-awareness.0.memory=73717
yarn.nodemanager.numa-awareness.0.cpus=4
yarn.nodemanager.numa-awareness.1.memory=73727
yarn.nodemanager.numa-awareness.1.cpus=4
</description>
<name>yarn.nodemanager.numa-awareness.node-ids</name>
<value></value>
</property>
<property>
<description>
The numactl command path which controls NUMA policy for processes or
shared memory.
</description>
<name>yarn.nodemanager.numa-awareness.numactl.cmd</name>
<value>/usr/bin/numactl</value>
</property>
</configuration>