001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.web;
019
020 import java.io.ByteArrayInputStream;
021 import java.io.DataInputStream;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.Collections;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.TreeMap;
028
029 import org.apache.hadoop.fs.ContentSummary;
030 import org.apache.hadoop.fs.FileChecksum;
031 import org.apache.hadoop.fs.FileStatus;
032 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
033 import org.apache.hadoop.fs.permission.FsPermission;
034 import org.apache.hadoop.hdfs.DFSUtil;
035 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
036 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
037 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
038 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
039 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
040 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
041 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
042 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
043 import org.apache.hadoop.io.MD5Hash;
044 import org.apache.hadoop.ipc.RemoteException;
045 import org.apache.hadoop.security.token.Token;
046 import org.apache.hadoop.security.token.TokenIdentifier;
047 import org.apache.hadoop.util.StringUtils;
048 import org.mortbay.util.ajax.JSON;
049
050 /** JSON Utilities */
051 public class JsonUtil {
052 private static final Object[] EMPTY_OBJECT_ARRAY = {};
053 private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
054
055 /** Convert a token object to a Json string. */
056 public static String toJsonString(final Token<? extends TokenIdentifier> token
057 ) throws IOException {
058 return toJsonString(Token.class, toJsonMap(token));
059 }
060
061 private static Map<String, Object> toJsonMap(
062 final Token<? extends TokenIdentifier> token) throws IOException {
063 if (token == null) {
064 return null;
065 }
066
067 final Map<String, Object> m = new TreeMap<String, Object>();
068 m.put("urlString", token.encodeToUrlString());
069 return m;
070 }
071
072 /** Convert a Json map to a Token. */
073 public static Token<? extends TokenIdentifier> toToken(
074 final Map<?, ?> m) throws IOException {
075 if (m == null) {
076 return null;
077 }
078
079 final Token<DelegationTokenIdentifier> token
080 = new Token<DelegationTokenIdentifier>();
081 token.decodeFromUrlString((String)m.get("urlString"));
082 return token;
083 }
084
085 /** Convert a Json map to a Token of DelegationTokenIdentifier. */
086 @SuppressWarnings("unchecked")
087 public static Token<DelegationTokenIdentifier> toDelegationToken(
088 final Map<?, ?> json) throws IOException {
089 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
090 return (Token<DelegationTokenIdentifier>)toToken(m);
091 }
092
093 /** Convert a Json map to a Token of BlockTokenIdentifier. */
094 @SuppressWarnings("unchecked")
095 private static Token<BlockTokenIdentifier> toBlockToken(
096 final Map<?, ?> m) throws IOException {
097 return (Token<BlockTokenIdentifier>)toToken(m);
098 }
099
100 /** Convert a Token[] to a JSON array. */
101 private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
102 ) throws IOException {
103 if (array == null) {
104 return null;
105 } else if (array.length == 0) {
106 return EMPTY_OBJECT_ARRAY;
107 } else {
108 final Object[] a = new Object[array.length];
109 for(int i = 0; i < array.length; i++) {
110 a[i] = toJsonMap(array[i]);
111 }
112 return a;
113 }
114 }
115
116 /** Convert a token object to a JSON string. */
117 public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
118 ) throws IOException {
119 if (tokens == null) {
120 return null;
121 }
122
123 final Map<String, Object> m = new TreeMap<String, Object>();
124 m.put(Token.class.getSimpleName(), toJsonArray(tokens));
125 return toJsonString(Token.class.getSimpleName() + "s", m);
126 }
127
128 /** Convert an Object[] to a List<Token<?>>. */
129 private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
130 if (objects == null) {
131 return null;
132 } else if (objects.length == 0) {
133 return Collections.emptyList();
134 } else {
135 final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
136 for(int i = 0; i < objects.length; i++) {
137 list.add(toToken((Map<?, ?>)objects[i]));
138 }
139 return list;
140 }
141 }
142
143 /** Convert a JSON map to a List<Token<?>>. */
144 public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
145 if (json == null) {
146 return null;
147 }
148
149 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
150 return toTokenList((Object[])m.get(Token.class.getSimpleName()));
151 }
152
153 /** Convert an exception object to a Json string. */
154 public static String toJsonString(final Exception e) {
155 final Map<String, Object> m = new TreeMap<String, Object>();
156 m.put("exception", e.getClass().getSimpleName());
157 m.put("message", e.getMessage());
158 m.put("javaClassName", e.getClass().getName());
159 return toJsonString(RemoteException.class, m);
160 }
161
162 /** Convert a Json map to a RemoteException. */
163 public static RemoteException toRemoteException(final Map<?, ?> json) {
164 final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
165 final String message = (String)m.get("message");
166 final String javaClassName = (String)m.get("javaClassName");
167 return new RemoteException(javaClassName, message);
168 }
169
170 private static String toJsonString(final Class<?> clazz, final Object value) {
171 return toJsonString(clazz.getSimpleName(), value);
172 }
173
174 /** Convert a key-value pair to a Json string. */
175 public static String toJsonString(final String key, final Object value) {
176 final Map<String, Object> m = new TreeMap<String, Object>();
177 m.put(key, value);
178 return JSON.toString(m);
179 }
180
181 /** Convert a FsPermission object to a string. */
182 private static String toString(final FsPermission permission) {
183 return String.format("%o", permission.toShort());
184 }
185
186 /** Convert a string to a FsPermission object. */
187 private static FsPermission toFsPermission(final String s) {
188 return new FsPermission(Short.parseShort(s, 8));
189 }
190
191 static enum PathType {
192 FILE, DIRECTORY, SYMLINK;
193
194 static PathType valueOf(HdfsFileStatus status) {
195 return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
196 }
197 }
198
199 /** Convert a HdfsFileStatus object to a Json string. */
200 public static String toJsonString(final HdfsFileStatus status,
201 boolean includeType) {
202 if (status == null) {
203 return null;
204 }
205 final Map<String, Object> m = new TreeMap<String, Object>();
206 m.put("pathSuffix", status.getLocalName());
207 m.put("type", PathType.valueOf(status));
208 if (status.isSymlink()) {
209 m.put("symlink", status.getSymlink());
210 }
211
212 m.put("length", status.getLen());
213 m.put("owner", status.getOwner());
214 m.put("group", status.getGroup());
215 m.put("permission", toString(status.getPermission()));
216 m.put("accessTime", status.getAccessTime());
217 m.put("modificationTime", status.getModificationTime());
218 m.put("blockSize", status.getBlockSize());
219 m.put("replication", status.getReplication());
220 return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
221 }
222
223 /** Convert a Json map to a HdfsFileStatus object. */
224 public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
225 if (json == null) {
226 return null;
227 }
228
229 final Map<?, ?> m = includesType ?
230 (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
231 final String localName = (String) m.get("pathSuffix");
232 final PathType type = PathType.valueOf((String) m.get("type"));
233 final byte[] symlink = type != PathType.SYMLINK? null
234 : DFSUtil.string2Bytes((String)m.get("symlink"));
235
236 final long len = (Long) m.get("length");
237 final String owner = (String) m.get("owner");
238 final String group = (String) m.get("group");
239 final FsPermission permission = toFsPermission((String) m.get("permission"));
240 final long aTime = (Long) m.get("accessTime");
241 final long mTime = (Long) m.get("modificationTime");
242 final long blockSize = (Long) m.get("blockSize");
243 final short replication = (short) (long) (Long) m.get("replication");
244 return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
245 blockSize, mTime, aTime, permission, owner, group,
246 symlink, DFSUtil.string2Bytes(localName));
247 }
248
249 /** Convert an ExtendedBlock to a Json map. */
250 private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
251 if (extendedblock == null) {
252 return null;
253 }
254
255 final Map<String, Object> m = new TreeMap<String, Object>();
256 m.put("blockPoolId", extendedblock.getBlockPoolId());
257 m.put("blockId", extendedblock.getBlockId());
258 m.put("numBytes", extendedblock.getNumBytes());
259 m.put("generationStamp", extendedblock.getGenerationStamp());
260 return m;
261 }
262
263 /** Convert a Json map to an ExtendedBlock object. */
264 private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
265 if (m == null) {
266 return null;
267 }
268
269 final String blockPoolId = (String)m.get("blockPoolId");
270 final long blockId = (Long)m.get("blockId");
271 final long numBytes = (Long)m.get("numBytes");
272 final long generationStamp = (Long)m.get("generationStamp");
273 return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
274 }
275
276 /** Convert a DatanodeInfo to a Json map. */
277 private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
278 if (datanodeinfo == null) {
279 return null;
280 }
281
282 final Map<String, Object> m = new TreeMap<String, Object>();
283 m.put("ipAddr", datanodeinfo.getIpAddr());
284 m.put("hostName", datanodeinfo.getHostName());
285 m.put("storageID", datanodeinfo.getStorageID());
286 m.put("xferPort", datanodeinfo.getXferPort());
287 m.put("infoPort", datanodeinfo.getInfoPort());
288 m.put("ipcPort", datanodeinfo.getIpcPort());
289
290 m.put("capacity", datanodeinfo.getCapacity());
291 m.put("dfsUsed", datanodeinfo.getDfsUsed());
292 m.put("remaining", datanodeinfo.getRemaining());
293 m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
294 m.put("lastUpdate", datanodeinfo.getLastUpdate());
295 m.put("xceiverCount", datanodeinfo.getXceiverCount());
296 m.put("networkLocation", datanodeinfo.getNetworkLocation());
297 m.put("adminState", datanodeinfo.getAdminState().name());
298 return m;
299 }
300
301 /** Convert a Json map to an DatanodeInfo object. */
302 private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
303 if (m == null) {
304 return null;
305 }
306
307 return new DatanodeInfo(
308 (String)m.get("ipAddr"),
309 (String)m.get("hostName"),
310 (String)m.get("storageID"),
311 (int)(long)(Long)m.get("xferPort"),
312 (int)(long)(Long)m.get("infoPort"),
313 (int)(long)(Long)m.get("ipcPort"),
314
315 (Long)m.get("capacity"),
316 (Long)m.get("dfsUsed"),
317 (Long)m.get("remaining"),
318 (Long)m.get("blockPoolUsed"),
319 (Long)m.get("lastUpdate"),
320 (int)(long)(Long)m.get("xceiverCount"),
321 (String)m.get("networkLocation"),
322 AdminStates.valueOf((String)m.get("adminState")));
323 }
324
325 /** Convert a DatanodeInfo[] to a Json array. */
326 private static Object[] toJsonArray(final DatanodeInfo[] array) {
327 if (array == null) {
328 return null;
329 } else if (array.length == 0) {
330 return EMPTY_OBJECT_ARRAY;
331 } else {
332 final Object[] a = new Object[array.length];
333 for(int i = 0; i < array.length; i++) {
334 a[i] = toJsonMap(array[i]);
335 }
336 return a;
337 }
338 }
339
340 /** Convert an Object[] to a DatanodeInfo[]. */
341 private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
342 if (objects == null) {
343 return null;
344 } else if (objects.length == 0) {
345 return EMPTY_DATANODE_INFO_ARRAY;
346 } else {
347 final DatanodeInfo[] array = new DatanodeInfo[objects.length];
348 for(int i = 0; i < array.length; i++) {
349 array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
350 }
351 return array;
352 }
353 }
354
355 /** Convert a LocatedBlock to a Json map. */
356 private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
357 ) throws IOException {
358 if (locatedblock == null) {
359 return null;
360 }
361
362 final Map<String, Object> m = new TreeMap<String, Object>();
363 m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
364 m.put("isCorrupt", locatedblock.isCorrupt());
365 m.put("startOffset", locatedblock.getStartOffset());
366 m.put("block", toJsonMap(locatedblock.getBlock()));
367 m.put("locations", toJsonArray(locatedblock.getLocations()));
368 return m;
369 }
370
371 /** Convert a Json map to LocatedBlock. */
372 private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
373 if (m == null) {
374 return null;
375 }
376
377 final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
378 final DatanodeInfo[] locations = toDatanodeInfoArray(
379 (Object[])m.get("locations"));
380 final long startOffset = (Long)m.get("startOffset");
381 final boolean isCorrupt = (Boolean)m.get("isCorrupt");
382
383 final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
384 locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
385 return locatedblock;
386 }
387
388 /** Convert a LocatedBlock[] to a Json array. */
389 private static Object[] toJsonArray(final List<LocatedBlock> array
390 ) throws IOException {
391 if (array == null) {
392 return null;
393 } else if (array.size() == 0) {
394 return EMPTY_OBJECT_ARRAY;
395 } else {
396 final Object[] a = new Object[array.size()];
397 for(int i = 0; i < array.size(); i++) {
398 a[i] = toJsonMap(array.get(i));
399 }
400 return a;
401 }
402 }
403
404 /** Convert an Object[] to a List of LocatedBlock. */
405 private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
406 ) throws IOException {
407 if (objects == null) {
408 return null;
409 } else if (objects.length == 0) {
410 return Collections.emptyList();
411 } else {
412 final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
413 for(int i = 0; i < objects.length; i++) {
414 list.add(toLocatedBlock((Map<?, ?>)objects[i]));
415 }
416 return list;
417 }
418 }
419
420 /** Convert LocatedBlocks to a Json string. */
421 public static String toJsonString(final LocatedBlocks locatedblocks
422 ) throws IOException {
423 if (locatedblocks == null) {
424 return null;
425 }
426
427 final Map<String, Object> m = new TreeMap<String, Object>();
428 m.put("fileLength", locatedblocks.getFileLength());
429 m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
430
431 m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
432 m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
433 m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
434 return toJsonString(LocatedBlocks.class, m);
435 }
436
437 /** Convert a Json map to LocatedBlock. */
438 public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
439 ) throws IOException {
440 if (json == null) {
441 return null;
442 }
443
444 final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
445 final long fileLength = (Long)m.get("fileLength");
446 final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
447 final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
448 (Object[])m.get("locatedBlocks"));
449 final LocatedBlock lastLocatedBlock = toLocatedBlock(
450 (Map<?, ?>)m.get("lastLocatedBlock"));
451 final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
452 return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
453 lastLocatedBlock, isLastBlockComplete);
454 }
455
456 /** Convert a ContentSummary to a Json string. */
457 public static String toJsonString(final ContentSummary contentsummary) {
458 if (contentsummary == null) {
459 return null;
460 }
461
462 final Map<String, Object> m = new TreeMap<String, Object>();
463 m.put("length", contentsummary.getLength());
464 m.put("fileCount", contentsummary.getFileCount());
465 m.put("directoryCount", contentsummary.getDirectoryCount());
466 m.put("quota", contentsummary.getQuota());
467 m.put("spaceConsumed", contentsummary.getSpaceConsumed());
468 m.put("spaceQuota", contentsummary.getSpaceQuota());
469 return toJsonString(ContentSummary.class, m);
470 }
471
472 /** Convert a Json map to a ContentSummary. */
473 public static ContentSummary toContentSummary(final Map<?, ?> json) {
474 if (json == null) {
475 return null;
476 }
477
478 final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
479 final long length = (Long)m.get("length");
480 final long fileCount = (Long)m.get("fileCount");
481 final long directoryCount = (Long)m.get("directoryCount");
482 final long quota = (Long)m.get("quota");
483 final long spaceConsumed = (Long)m.get("spaceConsumed");
484 final long spaceQuota = (Long)m.get("spaceQuota");
485
486 return new ContentSummary(length, fileCount, directoryCount,
487 quota, spaceConsumed, spaceQuota);
488 }
489
490 /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
491 public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
492 if (checksum == null) {
493 return null;
494 }
495
496 final Map<String, Object> m = new TreeMap<String, Object>();
497 m.put("algorithm", checksum.getAlgorithmName());
498 m.put("length", checksum.getLength());
499 m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
500 return toJsonString(FileChecksum.class, m);
501 }
502
503 /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
504 public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
505 final Map<?, ?> json) throws IOException {
506 if (json == null) {
507 return null;
508 }
509
510 final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
511 final String algorithm = (String)m.get("algorithm");
512 final int length = (int)(long)(Long)m.get("length");
513 final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
514
515 final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
516 final MD5MD5CRC32FileChecksum checksum = new MD5MD5CRC32FileChecksum();
517 checksum.readFields(in);
518
519 //check algorithm name
520 if (!checksum.getAlgorithmName().equals(algorithm)) {
521 throw new IOException("Algorithm not matched. Expected " + algorithm
522 + ", Received " + checksum.getAlgorithmName());
523 }
524 //check length
525 if (length != checksum.getLength()) {
526 throw new IOException("Length not matched: length=" + length
527 + ", checksum.getLength()=" + checksum.getLength());
528 }
529
530 return checksum;
531 }
532 }