1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.hbase;
19
20 import java.io.IOException;
21 import java.lang.reflect.InvocationTargetException;
22 import java.lang.reflect.Method;
23 import java.util.Map;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.hbase.classification.InterfaceStability;
30 import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
31 import org.apache.hadoop.hbase.util.VersionInfo;
32 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
33
34 /**
35 * Adds HBase configuration files to a Configuration
36 */
37 @InterfaceAudience.Public
38 @InterfaceStability.Stable
39 public class HBaseConfiguration extends Configuration {
40
41 private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
42
43 /**
44 * Instantinating HBaseConfiguration() is deprecated. Please use
45 * HBaseConfiguration#create() to construct a plain Configuration
46 */
47 @Deprecated
48 public HBaseConfiguration() {
49 //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration
50 super();
51 addHbaseResources(this);
52 LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use"
53 + " HBaseConfiguration#create() to construct a plain Configuration");
54 }
55
56 /**
57 * Instantiating HBaseConfiguration() is deprecated. Please use
58 * HBaseConfiguration#create(conf) to construct a plain Configuration
59 */
60 @Deprecated
61 public HBaseConfiguration(final Configuration c) {
62 //TODO:replace with private constructor
63 this();
64 merge(this, c);
65 }
66
67 private static void checkDefaultsVersion(Configuration conf) {
68 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
69 String defaultsVersion = conf.get("hbase.defaults.for.version");
70 String thisVersion = VersionInfo.getVersion();
71 if (!thisVersion.equals(defaultsVersion)) {
72 throw new RuntimeException(
73 "hbase-default.xml file seems to be for an older version of HBase (" +
74 defaultsVersion + "), this version is " + thisVersion);
75 }
76 }
77
78 public static Configuration addHbaseResources(Configuration conf) {
79 conf.addResource("hbase-default.xml");
80 conf.addResource("hbase-site.xml");
81
82 checkDefaultsVersion(conf);
83 HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
84 return conf;
85 }
86
87 /**
88 * Creates a Configuration with HBase resources
89 * @return a Configuration with HBase resources
90 */
91 public static Configuration create() {
92 Configuration conf = new Configuration();
93 // In case HBaseConfiguration is loaded from a different classloader than
94 // Configuration, conf needs to be set with appropriate class loader to resolve
95 // HBase resources.
96 conf.setClassLoader(HBaseConfiguration.class.getClassLoader());
97 return addHbaseResources(conf);
98 }
99
100 /**
101 * @param that Configuration to clone.
102 * @return a Configuration created with the hbase-*.xml files plus
103 * the given configuration.
104 */
105 public static Configuration create(final Configuration that) {
106 Configuration conf = create();
107 merge(conf, that);
108 return conf;
109 }
110
111 /**
112 * Merge two configurations.
113 * @param destConf the configuration that will be overwritten with items
114 * from the srcConf
115 * @param srcConf the source configuration
116 **/
117 public static void merge(Configuration destConf, Configuration srcConf) {
118 for (Map.Entry<String, String> e : srcConf) {
119 destConf.set(e.getKey(), e.getValue());
120 }
121 }
122
123 /**
124 * Returns a subset of the configuration properties, matching the given key prefix.
125 * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix",
126 * the entry "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's
127 * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be
128 * included in the results, since it would show up as an entry with an empty key.
129 */
130 public static Configuration subset(Configuration srcConf, String prefix) {
131 Configuration newConf = new Configuration(false);
132 for (Map.Entry<String, String> entry : srcConf) {
133 if (entry.getKey().startsWith(prefix)) {
134 String newKey = entry.getKey().substring(prefix.length());
135 // avoid entries that would produce an empty key
136 if (!newKey.isEmpty()) {
137 newConf.set(newKey, entry.getValue());
138 }
139 }
140 }
141 return newConf;
142 }
143
144 /**
145 * Sets all the entries in the provided {@code Map<String, String>} as properties in the
146 * given {@code Configuration}. Each property will have the specified prefix prepended,
147 * so that the configuration entries are keyed by {@code prefix + entry.getKey()}.
148 */
149 public static void setWithPrefix(Configuration conf, String prefix,
150 Iterable<Map.Entry<String, String>> properties) {
151 for (Map.Entry<String, String> entry : properties) {
152 conf.set(prefix + entry.getKey(), entry.getValue());
153 }
154 }
155
156 /**
157 * @return whether to show HBase Configuration in servlet
158 */
159 public static boolean isShowConfInServlet() {
160 boolean isShowConf = false;
161 try {
162 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) {
163 isShowConf = true;
164 }
165 } catch (LinkageError e) {
166 // should we handle it more aggressively in addition to log the error?
167 LOG.warn("Error thrown: ", e);
168 } catch (ClassNotFoundException ce) {
169 LOG.debug("ClassNotFound: ConfServlet");
170 // ignore
171 }
172 return isShowConf;
173 }
174
175 /**
176 * Get the value of the <code>name</code> property as an <code>int</code>, possibly
177 * referring to the deprecated name of the configuration property.
178 * If no such property exists, the provided default value is returned,
179 * or if the specified value is not a valid <code>int</code>,
180 * then an error is thrown.
181 *
182 * @param name property name.
183 * @param deprecatedName a deprecatedName for the property to use
184 * if non-deprecated name is not used
185 * @param defaultValue default value.
186 * @throws NumberFormatException when the value is invalid
187 * @return property value as an <code>int</code>,
188 * or <code>defaultValue</code>.
189 */
190 // TODO: developer note: This duplicates the functionality of deprecated
191 // property support in Configuration in Hadoop 2. But since Hadoop-1 does not
192 // contain these changes, we will do our own as usual. Replace these when H2 is default.
193 public static int getInt(Configuration conf, String name,
194 String deprecatedName, int defaultValue) {
195 if (conf.get(deprecatedName) != null) {
196 LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\""
197 , deprecatedName, name));
198 return conf.getInt(deprecatedName, defaultValue);
199 } else {
200 return conf.getInt(name, defaultValue);
201 }
202 }
203
204 /**
205 * Get the password from the Configuration instance using the
206 * getPassword method if it exists. If not, then fall back to the
207 * general get method for configuration elements.
208 * @param conf configuration instance for accessing the passwords
209 * @param alias the name of the password element
210 * @param defPass the default password
211 * @return String password or default password
212 * @throws IOException
213 */
214 public static String getPassword(Configuration conf, String alias,
215 String defPass) throws IOException {
216 String passwd = null;
217 try {
218 Method m = Configuration.class.getMethod("getPassword", String.class);
219 char[] p = (char[]) m.invoke(conf, alias);
220 if (p != null) {
221 LOG.debug(String.format("Config option \"%s\" was found through" +
222 " the Configuration getPassword method.", alias));
223 passwd = new String(p);
224 }
225 else {
226 LOG.debug(String.format(
227 "Config option \"%s\" was not found. Using provided default value",
228 alias));
229 passwd = defPass;
230 }
231 } catch (NoSuchMethodException e) {
232 // this is a version of Hadoop where the credential
233 //provider API doesn't exist yet
234 LOG.debug(String.format(
235 "Credential.getPassword method is not available." +
236 " Falling back to configuration."));
237 passwd = conf.get(alias, defPass);
238 } catch (SecurityException e) {
239 throw new IOException(e.getMessage(), e);
240 } catch (IllegalAccessException e) {
241 throw new IOException(e.getMessage(), e);
242 } catch (IllegalArgumentException e) {
243 throw new IOException(e.getMessage(), e);
244 } catch (InvocationTargetException e) {
245 throw new IOException(e.getMessage(), e);
246 }
247 return passwd;
248 }
249
250 /**
251 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key
252 * to the base Configuration. Note that additional configuration properties may be needed
253 * for a remote cluster, so it is preferable to use
254 * {@link #createClusterConf(Configuration, String, String)}.
255 *
256 * @param baseConf the base configuration to use, containing prefixed override properties
257 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
258 *
259 * @return the merged configuration with override properties and cluster key applied
260 *
261 * @see #createClusterConf(Configuration, String, String)
262 */
263 public static Configuration createClusterConf(Configuration baseConf, String clusterKey)
264 throws IOException {
265 return createClusterConf(baseConf, clusterKey, null);
266 }
267
268 /**
269 * Generates a {@link Configuration} instance by applying property overrides prefixed by
270 * a cluster profile key to the base Configuration. Override properties are extracted by
271 * the {@link #subset(Configuration, String)} method, then the merged on top of the base
272 * Configuration and returned.
273 *
274 * @param baseConf the base configuration to use, containing prefixed override properties
275 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none
276 * @param overridePrefix the property key prefix to match for override properties,
277 * or {@code null} if none
278 * @return the merged configuration with override properties and cluster key applied
279 */
280 public static Configuration createClusterConf(Configuration baseConf, String clusterKey,
281 String overridePrefix) throws IOException {
282 Configuration clusterConf = HBaseConfiguration.create(baseConf);
283 if (clusterKey != null && !clusterKey.isEmpty()) {
284 applyClusterKeyToConf(clusterConf, clusterKey);
285 }
286
287 if (overridePrefix != null && !overridePrefix.isEmpty()) {
288 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix);
289 HBaseConfiguration.merge(clusterConf, clusterSubset);
290 }
291 return clusterConf;
292 }
293
294 /**
295 * Apply the settings in the given key to the given configuration, this is
296 * used to communicate with distant clusters
297 * @param conf configuration object to configure
298 * @param key string that contains the 3 required configuratins
299 * @throws IOException
300 */
301 private static void applyClusterKeyToConf(Configuration conf, String key)
302 throws IOException{
303 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key);
304 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString());
305 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort());
306 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent());
307 }
308
309 /**
310 * For debugging. Dump configurations to system output as xml format.
311 * Master and RS configurations can also be dumped using
312 * http services. e.g. "curl http://master:16010/dump"
313 */
314 public static void main(String[] args) throws Exception {
315 HBaseConfiguration.create().writeXml(System.out);
316 }
317 }