001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.protocol;
019
020 import java.util.Comparator;
021 import java.util.Map;
022 import java.util.SortedSet;
023 import java.util.TreeSet;
024
025 import org.apache.hadoop.classification.InterfaceAudience;
026
027 /**
028 * This class tracks changes in the layout version of HDFS.
029 *
030 * Layout version is changed for following reasons:
031 * <ol>
032 * <li>The layout of how namenode or datanode stores information
033 * on disk changes.</li>
034 * <li>A new operation code is added to the editlog.</li>
035 * <li>Modification such as format of a record, content of a record
036 * in editlog or fsimage.</li>
037 * </ol>
038 * <br>
039 * <b>How to update layout version:<br></b>
040 * When a change requires new layout version, please add an entry into
041 * {@link Feature} with a short enum name, new layout version and description
042 * of the change. Please see {@link Feature} for further details.
043 * <br>
044 */
045 @InterfaceAudience.Private
046 public class LayoutVersion {
047 /**
048 * Version in which HDFS-2991 was fixed. This bug caused OP_ADD to
049 * sometimes be skipped for append() calls. If we see such a case when
050 * loading the edits, but the version is known to have that bug, we
051 * workaround the issue. Otherwise we should consider it a corruption
052 * and bail.
053 */
054 public static final int BUGFIX_HDFS_2991_VERSION = -40;
055
056 /**
057 * The interface to be implemented by NameNode and DataNode layout features
058 */
059 public interface LayoutFeature {
060 public FeatureInfo getInfo();
061 }
062
063 /**
064 * Enums for features that change the layout version before rolling
065 * upgrade is supported.
066 * <br><br>
067 * To add a new layout version:
068 * <ul>
069 * <li>Define a new enum constant with a short enum name, the new layout version
070 * and description of the added feature.</li>
071 * <li>When adding a layout version with an ancestor that is not same as
072 * its immediate predecessor, use the constructor where a specific ancestor
073 * can be passed.
074 * </li>
075 * </ul>
076 */
077 public static enum Feature implements LayoutFeature {
078 NAMESPACE_QUOTA(-16, "Support for namespace quotas"),
079 FILE_ACCESS_TIME(-17, "Support for access time on files"),
080 DISKSPACE_QUOTA(-18, "Support for disk space quotas"),
081 STICKY_BIT(-19, "Support for sticky bits"),
082 APPEND_RBW_DIR(-20, "Datanode has \"rbw\" subdirectory for append"),
083 ATOMIC_RENAME(-21, "Support for atomic rename"),
084 CONCAT(-22, "Support for concat operation"),
085 SYMLINKS(-23, "Support for symbolic links"),
086 DELEGATION_TOKEN(-24, "Support for delegation tokens for security"),
087 FSIMAGE_COMPRESSION(-25, "Support for fsimage compression"),
088 FSIMAGE_CHECKSUM(-26, "Support checksum for fsimage"),
089 REMOVE_REL13_DISK_LAYOUT_SUPPORT(-27, "Remove support for 0.13 disk layout"),
090 EDITS_CHESKUM(-28, "Support checksum for editlog"),
091 UNUSED(-29, "Skipped version"),
092 FSIMAGE_NAME_OPTIMIZATION(-30, "Store only last part of path in fsimage"),
093 RESERVED_REL20_203(-31, -19, "Reserved for release 0.20.203", true,
094 DELEGATION_TOKEN),
095 RESERVED_REL20_204(-32, -31, "Reserved for release 0.20.204", true),
096 RESERVED_REL22(-33, -27, "Reserved for release 0.22", true),
097 RESERVED_REL23(-34, -30, "Reserved for release 0.23", true),
098 FEDERATION(-35, "Support for namenode federation"),
099 LEASE_REASSIGNMENT(-36, "Support for persisting lease holder reassignment"),
100 STORED_TXIDS(-37, "Transaction IDs are stored in edits log and image files"),
101 TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"),
102 EDITLOG_OP_OPTIMIZATION(-39,
103 "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"),
104 OPTIMIZE_PERSIST_BLOCKS(-40,
105 "Serialize block lists with delta-encoded variable length ints, " +
106 "add OP_UPDATE_BLOCKS"),
107 RESERVED_REL1_2_0(-41, -32, "Reserved for release 1.2.0", true, CONCAT),
108 ADD_INODE_ID(-42, -40, "Assign a unique inode id for each inode", false),
109 SNAPSHOT(-43, "Support for snapshot feature"),
110 RESERVED_REL1_3_0(-44, -41, "Reserved for release 1.3.0", true,
111 ADD_INODE_ID, SNAPSHOT, FSIMAGE_NAME_OPTIMIZATION),
112 OPTIMIZE_SNAPSHOT_INODES(-45, -43,
113 "Reduce snapshot inode memory footprint", false),
114 SEQUENTIAL_BLOCK_ID(-46, "Allocate block IDs sequentially and store " +
115 "block IDs in the edits log and image files"),
116 EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to "
117 + "enable rebuilding retry cache in case of HA failover"),
118 EDITLOG_ADD_BLOCK(-48, "Add new editlog that only records allocation of "
119 + "the new block instead of the entire block list"),
120 ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid."
121 + " Use distinct StorageUuid per storage directory."),
122 ADD_LAYOUT_FLAGS(-50, "Add support for layout flags."),
123 CACHING(-51, "Support for cache pools and path-based caching"),
124 // Hadoop 2.4.0
125 PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"),
126 EXTENDED_ACL(-53, "Extended ACL"),
127 RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true,
128 PROTOBUF_FORMAT, EXTENDED_ACL);
129
130 private final FeatureInfo info;
131
132 /**
133 * Feature that is added at layout version {@code lv} - 1.
134 * @param lv new layout version with the addition of this feature
135 * @param description description of the feature
136 */
137 Feature(final int lv, final String description) {
138 this(lv, lv + 1, description, false);
139 }
140
141 /**
142 * Feature that is added at layout version {@code ancestoryLV}.
143 * @param lv new layout version with the addition of this feature
144 * @param ancestorLV layout version from which the new lv is derived from.
145 * @param description description of the feature
146 * @param reserved true when this is a layout version reserved for previous
147 * version
148 * @param features set of features that are to be enabled for this version
149 */
150 Feature(final int lv, final int ancestorLV, final String description,
151 boolean reserved, Feature... features) {
152 info = new FeatureInfo(lv, ancestorLV, description, reserved, features);
153 }
154
155 @Override
156 public FeatureInfo getInfo() {
157 return info;
158 }
159 }
160
161 /** Feature information. */
162 public static class FeatureInfo {
163 private final int lv;
164 private final int ancestorLV;
165 private final String description;
166 private final boolean reserved;
167 private final LayoutFeature[] specialFeatures;
168
169 public FeatureInfo(final int lv, final int ancestorLV, final String description,
170 boolean reserved, LayoutFeature... specialFeatures) {
171 this.lv = lv;
172 this.ancestorLV = ancestorLV;
173 this.description = description;
174 this.reserved = reserved;
175 this.specialFeatures = specialFeatures;
176 }
177
178 /**
179 * Accessor method for feature layout version
180 * @return int lv value
181 */
182 public int getLayoutVersion() {
183 return lv;
184 }
185
186 /**
187 * Accessor method for feature ancestor layout version
188 * @return int ancestor LV value
189 */
190 public int getAncestorLayoutVersion() {
191 return ancestorLV;
192 }
193
194 /**
195 * Accessor method for feature description
196 * @return String feature description
197 */
198 public String getDescription() {
199 return description;
200 }
201
202 public boolean isReservedForOldRelease() {
203 return reserved;
204 }
205
206 public LayoutFeature[] getSpecialFeatures() {
207 return specialFeatures;
208 }
209 }
210
211 static class LayoutFeatureComparator implements Comparator<LayoutFeature> {
212 @Override
213 public int compare(LayoutFeature arg0, LayoutFeature arg1) {
214 return arg0.getInfo().getLayoutVersion()
215 - arg1.getInfo().getLayoutVersion();
216 }
217 }
218
219 public static void updateMap(Map<Integer, SortedSet<LayoutFeature>> map,
220 LayoutFeature[] features) {
221 // Go through all the enum constants and build a map of
222 // LayoutVersion <-> Set of all supported features in that LayoutVersion
223 for (LayoutFeature f : features) {
224 final FeatureInfo info = f.getInfo();
225 SortedSet<LayoutFeature> ancestorSet = map.get(info.getAncestorLayoutVersion());
226 if (ancestorSet == null) {
227 // Empty set
228 ancestorSet = new TreeSet<LayoutFeature>(new LayoutFeatureComparator());
229 map.put(info.getAncestorLayoutVersion(), ancestorSet);
230 }
231 SortedSet<LayoutFeature> featureSet = new TreeSet<LayoutFeature>(ancestorSet);
232 if (info.getSpecialFeatures() != null) {
233 for (LayoutFeature specialFeature : info.getSpecialFeatures()) {
234 featureSet.add(specialFeature);
235 }
236 }
237 featureSet.add(f);
238 map.put(info.getLayoutVersion(), featureSet);
239 }
240 }
241
242 /**
243 * Gets formatted string that describes {@link LayoutVersion} information.
244 */
245 public String getString(Map<Integer, SortedSet<LayoutFeature>> map,
246 LayoutFeature[] values) {
247 final StringBuilder buf = new StringBuilder();
248 buf.append("Feature List:\n");
249 for (LayoutFeature f : values) {
250 final FeatureInfo info = f.getInfo();
251 buf.append(f).append(" introduced in layout version ")
252 .append(info.getLayoutVersion()).append(" (")
253 .append(info.getDescription()).append(")\n");
254 }
255
256 buf.append("\n\nLayoutVersion and supported features:\n");
257 for (LayoutFeature f : values) {
258 final FeatureInfo info = f.getInfo();
259 buf.append(info.getLayoutVersion()).append(": ")
260 .append(map.get(info.getLayoutVersion())).append("\n");
261 }
262 return buf.toString();
263 }
264
265 /**
266 * Returns true if a given feature is supported in the given layout version
267 * @param map layout feature map
268 * @param f Feature
269 * @param lv LayoutVersion
270 * @return true if {@code f} is supported in layout version {@code lv}
271 */
272 public static boolean supports(Map<Integer, SortedSet<LayoutFeature>> map,
273 final LayoutFeature f, final int lv) {
274 final SortedSet<LayoutFeature> set = map.get(lv);
275 return set != null && set.contains(f);
276 }
277
278 /**
279 * Get the current layout version
280 */
281 public static int getCurrentLayoutVersion(LayoutFeature[] features) {
282 return getLastNonReservedFeature(features).getInfo().getLayoutVersion();
283 }
284
285 static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) {
286 for (int i = features.length -1; i >= 0; i--) {
287 final FeatureInfo info = features[i].getInfo();
288 if (!info.isReservedForOldRelease()) {
289 return features[i];
290 }
291 }
292 throw new AssertionError("All layout versions are reserved.");
293 }
294 }
295