001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.web;
019
020 import java.io.ByteArrayInputStream;
021 import java.io.DataInputStream;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.Collections;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.TreeMap;
028
029 import org.apache.hadoop.fs.ContentSummary;
030 import org.apache.hadoop.fs.FileChecksum;
031 import org.apache.hadoop.fs.FileStatus;
032 import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
033 import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
034 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
035 import org.apache.hadoop.fs.permission.FsPermission;
036 import org.apache.hadoop.hdfs.DFSUtil;
037 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
038 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
039 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
040 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
041 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
042 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
043 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
044 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
045 import org.apache.hadoop.ipc.RemoteException;
046 import org.apache.hadoop.security.token.Token;
047 import org.apache.hadoop.security.token.TokenIdentifier;
048 import org.apache.hadoop.util.DataChecksum;
049 import org.apache.hadoop.util.StringUtils;
050 import org.mortbay.util.ajax.JSON;
051
052 /** JSON Utilities */
053 public class JsonUtil {
054 private static final Object[] EMPTY_OBJECT_ARRAY = {};
055 private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
056
057 /** Convert a token object to a Json string. */
058 public static String toJsonString(final Token<? extends TokenIdentifier> token
059 ) throws IOException {
060 return toJsonString(Token.class, toJsonMap(token));
061 }
062
063 private static Map<String, Object> toJsonMap(
064 final Token<? extends TokenIdentifier> token) throws IOException {
065 if (token == null) {
066 return null;
067 }
068
069 final Map<String, Object> m = new TreeMap<String, Object>();
070 m.put("urlString", token.encodeToUrlString());
071 return m;
072 }
073
074 /** Convert a Json map to a Token. */
075 public static Token<? extends TokenIdentifier> toToken(
076 final Map<?, ?> m) throws IOException {
077 if (m == null) {
078 return null;
079 }
080
081 final Token<DelegationTokenIdentifier> token
082 = new Token<DelegationTokenIdentifier>();
083 token.decodeFromUrlString((String)m.get("urlString"));
084 return token;
085 }
086
087 /** Convert a Json map to a Token of DelegationTokenIdentifier. */
088 @SuppressWarnings("unchecked")
089 public static Token<DelegationTokenIdentifier> toDelegationToken(
090 final Map<?, ?> json) throws IOException {
091 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
092 return (Token<DelegationTokenIdentifier>)toToken(m);
093 }
094
095 /** Convert a Json map to a Token of BlockTokenIdentifier. */
096 @SuppressWarnings("unchecked")
097 private static Token<BlockTokenIdentifier> toBlockToken(
098 final Map<?, ?> m) throws IOException {
099 return (Token<BlockTokenIdentifier>)toToken(m);
100 }
101
102 /** Convert a Token[] to a JSON array. */
103 private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
104 ) throws IOException {
105 if (array == null) {
106 return null;
107 } else if (array.length == 0) {
108 return EMPTY_OBJECT_ARRAY;
109 } else {
110 final Object[] a = new Object[array.length];
111 for(int i = 0; i < array.length; i++) {
112 a[i] = toJsonMap(array[i]);
113 }
114 return a;
115 }
116 }
117
118 /** Convert a token object to a JSON string. */
119 public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
120 ) throws IOException {
121 if (tokens == null) {
122 return null;
123 }
124
125 final Map<String, Object> m = new TreeMap<String, Object>();
126 m.put(Token.class.getSimpleName(), toJsonArray(tokens));
127 return toJsonString(Token.class.getSimpleName() + "s", m);
128 }
129
130 /** Convert an Object[] to a List<Token<?>>. */
131 private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
132 if (objects == null) {
133 return null;
134 } else if (objects.length == 0) {
135 return Collections.emptyList();
136 } else {
137 final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
138 for(int i = 0; i < objects.length; i++) {
139 list.add(toToken((Map<?, ?>)objects[i]));
140 }
141 return list;
142 }
143 }
144
145 /** Convert a JSON map to a List<Token<?>>. */
146 public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
147 if (json == null) {
148 return null;
149 }
150
151 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
152 return toTokenList((Object[])m.get(Token.class.getSimpleName()));
153 }
154
155 /** Convert an exception object to a Json string. */
156 public static String toJsonString(final Exception e) {
157 final Map<String, Object> m = new TreeMap<String, Object>();
158 m.put("exception", e.getClass().getSimpleName());
159 m.put("message", e.getMessage());
160 m.put("javaClassName", e.getClass().getName());
161 return toJsonString(RemoteException.class, m);
162 }
163
164 /** Convert a Json map to a RemoteException. */
165 public static RemoteException toRemoteException(final Map<?, ?> json) {
166 final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
167 final String message = (String)m.get("message");
168 final String javaClassName = (String)m.get("javaClassName");
169 return new RemoteException(javaClassName, message);
170 }
171
172 private static String toJsonString(final Class<?> clazz, final Object value) {
173 return toJsonString(clazz.getSimpleName(), value);
174 }
175
176 /** Convert a key-value pair to a Json string. */
177 public static String toJsonString(final String key, final Object value) {
178 final Map<String, Object> m = new TreeMap<String, Object>();
179 m.put(key, value);
180 return JSON.toString(m);
181 }
182
183 /** Convert a FsPermission object to a string. */
184 private static String toString(final FsPermission permission) {
185 return String.format("%o", permission.toShort());
186 }
187
188 /** Convert a string to a FsPermission object. */
189 private static FsPermission toFsPermission(final String s) {
190 return new FsPermission(Short.parseShort(s, 8));
191 }
192
193 static enum PathType {
194 FILE, DIRECTORY, SYMLINK;
195
196 static PathType valueOf(HdfsFileStatus status) {
197 return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
198 }
199 }
200
201 /** Convert a HdfsFileStatus object to a Json string. */
202 public static String toJsonString(final HdfsFileStatus status,
203 boolean includeType) {
204 if (status == null) {
205 return null;
206 }
207 final Map<String, Object> m = new TreeMap<String, Object>();
208 m.put("pathSuffix", status.getLocalName());
209 m.put("type", PathType.valueOf(status));
210 if (status.isSymlink()) {
211 m.put("symlink", status.getSymlink());
212 }
213
214 m.put("length", status.getLen());
215 m.put("owner", status.getOwner());
216 m.put("group", status.getGroup());
217 m.put("permission", toString(status.getPermission()));
218 m.put("accessTime", status.getAccessTime());
219 m.put("modificationTime", status.getModificationTime());
220 m.put("blockSize", status.getBlockSize());
221 m.put("replication", status.getReplication());
222 return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
223 }
224
225 /** Convert a Json map to a HdfsFileStatus object. */
226 public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
227 if (json == null) {
228 return null;
229 }
230
231 final Map<?, ?> m = includesType ?
232 (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
233 final String localName = (String) m.get("pathSuffix");
234 final PathType type = PathType.valueOf((String) m.get("type"));
235 final byte[] symlink = type != PathType.SYMLINK? null
236 : DFSUtil.string2Bytes((String)m.get("symlink"));
237
238 final long len = (Long) m.get("length");
239 final String owner = (String) m.get("owner");
240 final String group = (String) m.get("group");
241 final FsPermission permission = toFsPermission((String) m.get("permission"));
242 final long aTime = (Long) m.get("accessTime");
243 final long mTime = (Long) m.get("modificationTime");
244 final long blockSize = (Long) m.get("blockSize");
245 final short replication = (short) (long) (Long) m.get("replication");
246 return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
247 blockSize, mTime, aTime, permission, owner, group,
248 symlink, DFSUtil.string2Bytes(localName));
249 }
250
251 /** Convert an ExtendedBlock to a Json map. */
252 private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
253 if (extendedblock == null) {
254 return null;
255 }
256
257 final Map<String, Object> m = new TreeMap<String, Object>();
258 m.put("blockPoolId", extendedblock.getBlockPoolId());
259 m.put("blockId", extendedblock.getBlockId());
260 m.put("numBytes", extendedblock.getNumBytes());
261 m.put("generationStamp", extendedblock.getGenerationStamp());
262 return m;
263 }
264
265 /** Convert a Json map to an ExtendedBlock object. */
266 private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
267 if (m == null) {
268 return null;
269 }
270
271 final String blockPoolId = (String)m.get("blockPoolId");
272 final long blockId = (Long)m.get("blockId");
273 final long numBytes = (Long)m.get("numBytes");
274 final long generationStamp = (Long)m.get("generationStamp");
275 return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
276 }
277
278 /** Convert a DatanodeInfo to a Json map. */
279 private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
280 if (datanodeinfo == null) {
281 return null;
282 }
283
284 final Map<String, Object> m = new TreeMap<String, Object>();
285 m.put("ipAddr", datanodeinfo.getIpAddr());
286 m.put("hostName", datanodeinfo.getHostName());
287 m.put("storageID", datanodeinfo.getStorageID());
288 m.put("xferPort", datanodeinfo.getXferPort());
289 m.put("infoPort", datanodeinfo.getInfoPort());
290 m.put("ipcPort", datanodeinfo.getIpcPort());
291
292 m.put("capacity", datanodeinfo.getCapacity());
293 m.put("dfsUsed", datanodeinfo.getDfsUsed());
294 m.put("remaining", datanodeinfo.getRemaining());
295 m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
296 m.put("lastUpdate", datanodeinfo.getLastUpdate());
297 m.put("xceiverCount", datanodeinfo.getXceiverCount());
298 m.put("networkLocation", datanodeinfo.getNetworkLocation());
299 m.put("adminState", datanodeinfo.getAdminState().name());
300 return m;
301 }
302
303 /** Convert a Json map to an DatanodeInfo object. */
304 private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
305 if (m == null) {
306 return null;
307 }
308
309 return new DatanodeInfo(
310 (String)m.get("ipAddr"),
311 (String)m.get("hostName"),
312 (String)m.get("storageID"),
313 (int)(long)(Long)m.get("xferPort"),
314 (int)(long)(Long)m.get("infoPort"),
315 (int)(long)(Long)m.get("ipcPort"),
316
317 (Long)m.get("capacity"),
318 (Long)m.get("dfsUsed"),
319 (Long)m.get("remaining"),
320 (Long)m.get("blockPoolUsed"),
321 (Long)m.get("lastUpdate"),
322 (int)(long)(Long)m.get("xceiverCount"),
323 (String)m.get("networkLocation"),
324 AdminStates.valueOf((String)m.get("adminState")));
325 }
326
327 /** Convert a DatanodeInfo[] to a Json array. */
328 private static Object[] toJsonArray(final DatanodeInfo[] array) {
329 if (array == null) {
330 return null;
331 } else if (array.length == 0) {
332 return EMPTY_OBJECT_ARRAY;
333 } else {
334 final Object[] a = new Object[array.length];
335 for(int i = 0; i < array.length; i++) {
336 a[i] = toJsonMap(array[i]);
337 }
338 return a;
339 }
340 }
341
342 /** Convert an Object[] to a DatanodeInfo[]. */
343 private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
344 if (objects == null) {
345 return null;
346 } else if (objects.length == 0) {
347 return EMPTY_DATANODE_INFO_ARRAY;
348 } else {
349 final DatanodeInfo[] array = new DatanodeInfo[objects.length];
350 for(int i = 0; i < array.length; i++) {
351 array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
352 }
353 return array;
354 }
355 }
356
357 /** Convert a LocatedBlock to a Json map. */
358 private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
359 ) throws IOException {
360 if (locatedblock == null) {
361 return null;
362 }
363
364 final Map<String, Object> m = new TreeMap<String, Object>();
365 m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
366 m.put("isCorrupt", locatedblock.isCorrupt());
367 m.put("startOffset", locatedblock.getStartOffset());
368 m.put("block", toJsonMap(locatedblock.getBlock()));
369 m.put("locations", toJsonArray(locatedblock.getLocations()));
370 return m;
371 }
372
373 /** Convert a Json map to LocatedBlock. */
374 private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
375 if (m == null) {
376 return null;
377 }
378
379 final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
380 final DatanodeInfo[] locations = toDatanodeInfoArray(
381 (Object[])m.get("locations"));
382 final long startOffset = (Long)m.get("startOffset");
383 final boolean isCorrupt = (Boolean)m.get("isCorrupt");
384
385 final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
386 locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
387 return locatedblock;
388 }
389
390 /** Convert a LocatedBlock[] to a Json array. */
391 private static Object[] toJsonArray(final List<LocatedBlock> array
392 ) throws IOException {
393 if (array == null) {
394 return null;
395 } else if (array.size() == 0) {
396 return EMPTY_OBJECT_ARRAY;
397 } else {
398 final Object[] a = new Object[array.size()];
399 for(int i = 0; i < array.size(); i++) {
400 a[i] = toJsonMap(array.get(i));
401 }
402 return a;
403 }
404 }
405
406 /** Convert an Object[] to a List of LocatedBlock. */
407 private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
408 ) throws IOException {
409 if (objects == null) {
410 return null;
411 } else if (objects.length == 0) {
412 return Collections.emptyList();
413 } else {
414 final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
415 for(int i = 0; i < objects.length; i++) {
416 list.add(toLocatedBlock((Map<?, ?>)objects[i]));
417 }
418 return list;
419 }
420 }
421
422 /** Convert LocatedBlocks to a Json string. */
423 public static String toJsonString(final LocatedBlocks locatedblocks
424 ) throws IOException {
425 if (locatedblocks == null) {
426 return null;
427 }
428
429 final Map<String, Object> m = new TreeMap<String, Object>();
430 m.put("fileLength", locatedblocks.getFileLength());
431 m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
432
433 m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
434 m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
435 m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
436 return toJsonString(LocatedBlocks.class, m);
437 }
438
439 /** Convert a Json map to LocatedBlock. */
440 public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
441 ) throws IOException {
442 if (json == null) {
443 return null;
444 }
445
446 final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
447 final long fileLength = (Long)m.get("fileLength");
448 final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
449 final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
450 (Object[])m.get("locatedBlocks"));
451 final LocatedBlock lastLocatedBlock = toLocatedBlock(
452 (Map<?, ?>)m.get("lastLocatedBlock"));
453 final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
454 return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
455 lastLocatedBlock, isLastBlockComplete);
456 }
457
458 /** Convert a ContentSummary to a Json string. */
459 public static String toJsonString(final ContentSummary contentsummary) {
460 if (contentsummary == null) {
461 return null;
462 }
463
464 final Map<String, Object> m = new TreeMap<String, Object>();
465 m.put("length", contentsummary.getLength());
466 m.put("fileCount", contentsummary.getFileCount());
467 m.put("directoryCount", contentsummary.getDirectoryCount());
468 m.put("quota", contentsummary.getQuota());
469 m.put("spaceConsumed", contentsummary.getSpaceConsumed());
470 m.put("spaceQuota", contentsummary.getSpaceQuota());
471 return toJsonString(ContentSummary.class, m);
472 }
473
474 /** Convert a Json map to a ContentSummary. */
475 public static ContentSummary toContentSummary(final Map<?, ?> json) {
476 if (json == null) {
477 return null;
478 }
479
480 final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
481 final long length = (Long)m.get("length");
482 final long fileCount = (Long)m.get("fileCount");
483 final long directoryCount = (Long)m.get("directoryCount");
484 final long quota = (Long)m.get("quota");
485 final long spaceConsumed = (Long)m.get("spaceConsumed");
486 final long spaceQuota = (Long)m.get("spaceQuota");
487
488 return new ContentSummary(length, fileCount, directoryCount,
489 quota, spaceConsumed, spaceQuota);
490 }
491
492 /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
493 public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
494 if (checksum == null) {
495 return null;
496 }
497
498 final Map<String, Object> m = new TreeMap<String, Object>();
499 m.put("algorithm", checksum.getAlgorithmName());
500 m.put("length", checksum.getLength());
501 m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
502 return toJsonString(FileChecksum.class, m);
503 }
504
505 /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
506 public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
507 final Map<?, ?> json) throws IOException {
508 if (json == null) {
509 return null;
510 }
511
512 final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
513 final String algorithm = (String)m.get("algorithm");
514 final int length = (int)(long)(Long)m.get("length");
515 final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
516
517 final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
518 final DataChecksum.Type crcType =
519 MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
520 final MD5MD5CRC32FileChecksum checksum;
521
522 // Recreate what DFSClient would have returned.
523 switch(crcType) {
524 case CRC32:
525 checksum = new MD5MD5CRC32GzipFileChecksum();
526 break;
527 case CRC32C:
528 checksum = new MD5MD5CRC32CastagnoliFileChecksum();
529 break;
530 default:
531 throw new IOException("Unknown algorithm: " + algorithm);
532 }
533 checksum.readFields(in);
534
535 //check algorithm name
536 if (!checksum.getAlgorithmName().equals(algorithm)) {
537 throw new IOException("Algorithm not matched. Expected " + algorithm
538 + ", Received " + checksum.getAlgorithmName());
539 }
540 //check length
541 if (length != checksum.getLength()) {
542 throw new IOException("Length not matched: length=" + length
543 + ", checksum.getLength()=" + checksum.getLength());
544 }
545
546 return checksum;
547 }
548 }