1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.hbase.snapshot;
19
20 import java.io.IOException;
21 import java.util.Collections;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FSDataInputStream;
27 import org.apache.hadoop.fs.FSDataOutputStream;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.fs.permission.FsPermission;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
33 import org.apache.hadoop.hbase.security.User;
34 import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
35 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
36 import org.apache.hadoop.hbase.util.FSUtils;
37
38 /**
39 * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
40 * <p>
41 * Snapshots are laid out on disk like this:
42 *
43 * <pre>
44 * /hbase/.snapshots
45 * /.tmp <---- working directory
46 * /[snapshot name] <----- completed snapshot
47 * </pre>
48 *
49 * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
50 * signified by '...' on the same directory depth).
51 *
52 * <pre>
53 * /hbase/.snapshots/completed
54 * .snapshotinfo <--- Description of the snapshot
55 * .tableinfo <--- Copy of the tableinfo
56 * /.logs
57 * /[server_name]
58 * /... [log files]
59 * ...
60 * /[region name] <---- All the region's information
61 * .regioninfo <---- Copy of the HRegionInfo
62 * /[column family name]
63 * /[hfile name] <--- name of the hfile in the real region
64 * ...
65 * ...
66 * ...
67 * </pre>
68 *
69 * Utility methods in this class are useful for getting the correct locations for different parts of
70 * the snapshot, as well as moving completed snapshots into place (see
71 * {@link #completeSnapshot}, and writing the
72 * {@link SnapshotDescription} to the working snapshot directory.
73 */
74 public class SnapshotDescriptionUtils {
75
76 /**
77 * Filter that only accepts completed snapshot directories
78 */
79 public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
80
81 /**
82 * @param fs
83 */
84 public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
85 super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
86 }
87 }
88
89 private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
90 /**
91 * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
92 * which we may need to read in differently.
93 */
94 public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
95
96 // snapshot directory constants
97 /**
98 * The file contains the snapshot basic information and it is under the directory of a snapshot.
99 */
100 public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
101
102 /** Temporary directory under the snapshot directory to store in-progress snapshots */
103 public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
104 // snapshot operation values
105 /** Default value if no start time is specified */
106 public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
107
108
109 public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis";
110
111 /** By default, wait 300 seconds for a snapshot to complete */
112 public static final long DEFAULT_MAX_WAIT_TIME = 60000 * 5 ;
113
114
115 /**
116 * By default, check to see if the snapshot is complete (ms)
117 * @deprecated Use {@link #DEFAULT_MAX_WAIT_TIME} instead.
118 * */
119 @Deprecated
120 public static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000 * 5;
121
122 /**
123 * Conf key for # of ms elapsed before injecting a snapshot timeout error when waiting for
124 * completion.
125 * @deprecated Use {@link #MASTER_SNAPSHOT_TIMEOUT_MILLIS} instead.
126 */
127 @Deprecated
128 public static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
129
130 private SnapshotDescriptionUtils() {
131 // private constructor for utility class
132 }
133
134 /**
135 * @param conf {@link Configuration} from which to check for the timeout
136 * @param type type of snapshot being taken
137 * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration
138 * @return the max amount of time the master should wait for a snapshot to complete
139 */
140 public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type,
141 long defaultMaxWaitTime) {
142 String confKey;
143 switch (type) {
144 case DISABLED:
145 default:
146 confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS;
147 }
148 return Math.max(conf.getLong(confKey, defaultMaxWaitTime),
149 conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, defaultMaxWaitTime));
150 }
151
152 /**
153 * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
154 * ${hbase.rootdir}/.snapshot
155 * @param rootDir hbase root directory
156 * @return the base directory in which all snapshots are kept
157 */
158 public static Path getSnapshotRootDir(final Path rootDir) {
159 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
160 }
161
162 /**
163 * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
164 * directory and all the data files for a snapshot are kept under this directory.
165 * @param snapshot snapshot being taken
166 * @param rootDir hbase root directory
167 * @return the final directory for the completed snapshot
168 */
169 public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
170 return getCompletedSnapshotDir(snapshot.getName(), rootDir);
171 }
172
173 /**
174 * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
175 * directory and all the data files for a snapshot are kept under this directory.
176 * @param snapshotName name of the snapshot being taken
177 * @param rootDir hbase root directory
178 * @return the final directory for the completed snapshot
179 */
180 public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
181 return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
182 }
183
184 /**
185 * Get the general working directory for snapshots - where they are built, where they are
186 * temporarily copied on export, etc.
187 * @param rootDir root directory of the HBase installation
188 * @return Path to the snapshot tmp directory, relative to the passed root directory
189 */
190 public static Path getWorkingSnapshotDir(final Path rootDir) {
191 return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
192 }
193
194 /**
195 * Get the directory to build a snapshot, before it is finalized
196 * @param snapshot snapshot that will be built
197 * @param rootDir root directory of the hbase installation
198 * @return {@link Path} where one can build a snapshot
199 */
200 public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
201 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
202 }
203
204 /**
205 * Get the directory to build a snapshot, before it is finalized
206 * @param snapshotName name of the snapshot
207 * @param rootDir root directory of the hbase installation
208 * @return {@link Path} where one can build a snapshot
209 */
210 public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
211 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
212 }
213
214 /**
215 * Get the directory to store the snapshot instance
216 * @param snapshotsDir hbase-global directory for storing all snapshots
217 * @param snapshotName name of the snapshot to take
218 * @return the final directory for the completed snapshot
219 */
220 private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
221 return new Path(snapshotsDir, snapshotName);
222 }
223
224 /**
225 * @param rootDir hbase root directory
226 * @return the directory for all completed snapshots;
227 */
228 public static final Path getSnapshotsDir(Path rootDir) {
229 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
230 }
231
232 /**
233 * Convert the passed snapshot description into a 'full' snapshot description based on default
234 * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
235 * supplied to their default values.
236 * @param snapshot general snapshot descriptor
237 * @param conf Configuration to read configured snapshot defaults if snapshot is not complete
238 * @return a valid snapshot description
239 * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete
240 * {@link SnapshotDescription}.
241 */
242 public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf)
243 throws IllegalArgumentException {
244 if (!snapshot.hasTable()) {
245 throw new IllegalArgumentException(
246 "Descriptor doesn't apply to a table, so we can't build it.");
247 }
248
249 // set the creation time, if one hasn't been set
250 long time = snapshot.getCreationTime();
251 if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) {
252 time = EnvironmentEdgeManager.currentTime();
253 LOG.debug("Creation time not specified, setting to:" + time + " (current time:"
254 + EnvironmentEdgeManager.currentTime() + ").");
255 SnapshotDescription.Builder builder = snapshot.toBuilder();
256 builder.setCreationTime(time);
257 snapshot = builder.build();
258 }
259 return snapshot;
260 }
261
262 /**
263 * Write the snapshot description into the working directory of a snapshot
264 * @param snapshot description of the snapshot being taken
265 * @param workingDir working directory of the snapshot
266 * @param fs {@link FileSystem} on which the snapshot should be taken
267 * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
268 * failure
269 */
270 public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
271 throws IOException {
272 FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
273 HConstants.DATA_FILE_UMASK_KEY);
274 Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
275 try {
276 FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
277 try {
278 snapshot.writeTo(out);
279 } finally {
280 out.close();
281 }
282 } catch (IOException e) {
283 // if we get an exception, try to remove the snapshot info
284 if (!fs.delete(snapshotInfo, false)) {
285 String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
286 LOG.error(msg);
287 throw new IOException(msg);
288 }
289 }
290 }
291
292 /**
293 * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
294 * @param fs filesystem where the snapshot was taken
295 * @param snapshotDir directory where the snapshot was stored
296 * @return the stored snapshot description
297 * @throws CorruptedSnapshotException if the
298 * snapshot cannot be read
299 */
300 public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
301 throws CorruptedSnapshotException {
302 Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
303 try {
304 FSDataInputStream in = null;
305 try {
306 in = fs.open(snapshotInfo);
307 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
308 return desc;
309 } finally {
310 if (in != null) in.close();
311 }
312 } catch (IOException e) {
313 throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
314 }
315 }
316
317 /**
318 * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
319 * as 'complete'.
320 * @param snapshot description of the snapshot being tabken
321 * @param rootdir root directory of the hbase installation
322 * @param workingDir directory where the in progress snapshot was built
323 * @param fs {@link FileSystem} where the snapshot was built
324 * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
325 * snapshot could not be moved
326 * @throws IOException the filesystem could not be reached
327 */
328 public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
329 FileSystem fs) throws SnapshotCreationException, IOException {
330 Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
331 LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
332 + finishedDir);
333 if (!fs.rename(workingDir, finishedDir)) {
334 throw new SnapshotCreationException("Failed to move working directory(" + workingDir
335 + ") to completed directory(" + finishedDir + ").", snapshot);
336 }
337 }
338
339 /**
340 * Check if the user is this table snapshot's owner
341 * @param snapshot the table snapshot description
342 * @param user the user
343 * @return true if the user is the owner of the snapshot,
344 * false otherwise or the snapshot owner field is not present.
345 */
346 public static boolean isSnapshotOwner(final SnapshotDescription snapshot, final User user) {
347 if (user == null) return false;
348 if (!snapshot.hasOwner()) return false;
349 return snapshot.getOwner().equals(user.getShortName());
350 }
351 }