001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.web;
019
020 import org.apache.hadoop.fs.*;
021 import org.apache.hadoop.fs.permission.FsPermission;
022 import org.apache.hadoop.hdfs.DFSUtil;
023 import org.apache.hadoop.hdfs.protocol.*;
024 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
025 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
026 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
027 import org.apache.hadoop.hdfs.server.namenode.INodeId;
028 import org.apache.hadoop.ipc.RemoteException;
029 import org.apache.hadoop.security.token.Token;
030 import org.apache.hadoop.security.token.TokenIdentifier;
031 import org.apache.hadoop.util.DataChecksum;
032 import org.apache.hadoop.util.StringUtils;
033 import org.mortbay.util.ajax.JSON;
034
035 import java.io.ByteArrayInputStream;
036 import java.io.DataInputStream;
037 import java.io.IOException;
038 import java.util.*;
039
040 /** JSON Utilities */
041 public class JsonUtil {
042 private static final Object[] EMPTY_OBJECT_ARRAY = {};
043 private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
044
045 /** Convert a token object to a Json string. */
046 public static String toJsonString(final Token<? extends TokenIdentifier> token
047 ) throws IOException {
048 return toJsonString(Token.class, toJsonMap(token));
049 }
050
051 private static Map<String, Object> toJsonMap(
052 final Token<? extends TokenIdentifier> token) throws IOException {
053 if (token == null) {
054 return null;
055 }
056
057 final Map<String, Object> m = new TreeMap<String, Object>();
058 m.put("urlString", token.encodeToUrlString());
059 return m;
060 }
061
062 /** Convert a Json map to a Token. */
063 public static Token<? extends TokenIdentifier> toToken(
064 final Map<?, ?> m) throws IOException {
065 if (m == null) {
066 return null;
067 }
068
069 final Token<DelegationTokenIdentifier> token
070 = new Token<DelegationTokenIdentifier>();
071 token.decodeFromUrlString((String)m.get("urlString"));
072 return token;
073 }
074
075 /** Convert a Json map to a Token of DelegationTokenIdentifier. */
076 @SuppressWarnings("unchecked")
077 public static Token<DelegationTokenIdentifier> toDelegationToken(
078 final Map<?, ?> json) throws IOException {
079 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
080 return (Token<DelegationTokenIdentifier>)toToken(m);
081 }
082
083 /** Convert a Json map to a Token of BlockTokenIdentifier. */
084 @SuppressWarnings("unchecked")
085 private static Token<BlockTokenIdentifier> toBlockToken(
086 final Map<?, ?> m) throws IOException {
087 return (Token<BlockTokenIdentifier>)toToken(m);
088 }
089
090 /** Convert a Token[] to a JSON array. */
091 private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
092 ) throws IOException {
093 if (array == null) {
094 return null;
095 } else if (array.length == 0) {
096 return EMPTY_OBJECT_ARRAY;
097 } else {
098 final Object[] a = new Object[array.length];
099 for(int i = 0; i < array.length; i++) {
100 a[i] = toJsonMap(array[i]);
101 }
102 return a;
103 }
104 }
105
106 /** Convert a token object to a JSON string. */
107 public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
108 ) throws IOException {
109 if (tokens == null) {
110 return null;
111 }
112
113 final Map<String, Object> m = new TreeMap<String, Object>();
114 m.put(Token.class.getSimpleName(), toJsonArray(tokens));
115 return toJsonString(Token.class.getSimpleName() + "s", m);
116 }
117
118 /** Convert an Object[] to a List<Token<?>>. */
119 private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
120 if (objects == null) {
121 return null;
122 } else if (objects.length == 0) {
123 return Collections.emptyList();
124 } else {
125 final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
126 for(int i = 0; i < objects.length; i++) {
127 list.add(toToken((Map<?, ?>)objects[i]));
128 }
129 return list;
130 }
131 }
132
133 /** Convert a JSON map to a List<Token<?>>. */
134 public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
135 if (json == null) {
136 return null;
137 }
138
139 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
140 return toTokenList((Object[])m.get(Token.class.getSimpleName()));
141 }
142
143 /** Convert an exception object to a Json string. */
144 public static String toJsonString(final Exception e) {
145 final Map<String, Object> m = new TreeMap<String, Object>();
146 m.put("exception", e.getClass().getSimpleName());
147 m.put("message", e.getMessage());
148 m.put("javaClassName", e.getClass().getName());
149 return toJsonString(RemoteException.class, m);
150 }
151
152 /** Convert a Json map to a RemoteException. */
153 public static RemoteException toRemoteException(final Map<?, ?> json) {
154 final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
155 final String message = (String)m.get("message");
156 final String javaClassName = (String)m.get("javaClassName");
157 return new RemoteException(javaClassName, message);
158 }
159
160 private static String toJsonString(final Class<?> clazz, final Object value) {
161 return toJsonString(clazz.getSimpleName(), value);
162 }
163
164 /** Convert a key-value pair to a Json string. */
165 public static String toJsonString(final String key, final Object value) {
166 final Map<String, Object> m = new TreeMap<String, Object>();
167 m.put(key, value);
168 return JSON.toString(m);
169 }
170
171 /** Convert a FsPermission object to a string. */
172 private static String toString(final FsPermission permission) {
173 return String.format("%o", permission.toShort());
174 }
175
176 /** Convert a string to a FsPermission object. */
177 private static FsPermission toFsPermission(final String s) {
178 return new FsPermission(Short.parseShort(s, 8));
179 }
180
181 static enum PathType {
182 FILE, DIRECTORY, SYMLINK;
183
184 static PathType valueOf(HdfsFileStatus status) {
185 return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
186 }
187 }
188
189 /** Convert a HdfsFileStatus object to a Json string. */
190 public static String toJsonString(final HdfsFileStatus status,
191 boolean includeType) {
192 if (status == null) {
193 return null;
194 }
195 final Map<String, Object> m = new TreeMap<String, Object>();
196 m.put("pathSuffix", status.getLocalName());
197 m.put("type", PathType.valueOf(status));
198 if (status.isSymlink()) {
199 m.put("symlink", status.getSymlink());
200 }
201
202 m.put("length", status.getLen());
203 m.put("owner", status.getOwner());
204 m.put("group", status.getGroup());
205 m.put("permission", toString(status.getPermission()));
206 m.put("accessTime", status.getAccessTime());
207 m.put("modificationTime", status.getModificationTime());
208 m.put("blockSize", status.getBlockSize());
209 m.put("replication", status.getReplication());
210 m.put("fileId", status.getFileId());
211 m.put("childrenNum", status.getChildrenNum());
212 return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
213 }
214
215 /** Convert a Json map to a HdfsFileStatus object. */
216 public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
217 if (json == null) {
218 return null;
219 }
220
221 final Map<?, ?> m = includesType ?
222 (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
223 final String localName = (String) m.get("pathSuffix");
224 final PathType type = PathType.valueOf((String) m.get("type"));
225 final byte[] symlink = type != PathType.SYMLINK? null
226 : DFSUtil.string2Bytes((String)m.get("symlink"));
227
228 final long len = (Long) m.get("length");
229 final String owner = (String) m.get("owner");
230 final String group = (String) m.get("group");
231 final FsPermission permission = toFsPermission((String) m.get("permission"));
232 final long aTime = (Long) m.get("accessTime");
233 final long mTime = (Long) m.get("modificationTime");
234 final long blockSize = (Long) m.get("blockSize");
235 final short replication = (short) (long) (Long) m.get("replication");
236 final long fileId = m.containsKey("fileId") ? (Long) m.get("fileId")
237 : INodeId.GRANDFATHER_INODE_ID;
238 Long childrenNumLong = (Long) m.get("childrenNum");
239 final int childrenNum = (childrenNumLong == null) ? -1
240 : childrenNumLong.intValue();
241 return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
242 blockSize, mTime, aTime, permission, owner, group,
243 symlink, DFSUtil.string2Bytes(localName), fileId, childrenNum);
244 }
245
246 /** Convert an ExtendedBlock to a Json map. */
247 private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
248 if (extendedblock == null) {
249 return null;
250 }
251
252 final Map<String, Object> m = new TreeMap<String, Object>();
253 m.put("blockPoolId", extendedblock.getBlockPoolId());
254 m.put("blockId", extendedblock.getBlockId());
255 m.put("numBytes", extendedblock.getNumBytes());
256 m.put("generationStamp", extendedblock.getGenerationStamp());
257 return m;
258 }
259
260 /** Convert a Json map to an ExtendedBlock object. */
261 private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
262 if (m == null) {
263 return null;
264 }
265
266 final String blockPoolId = (String)m.get("blockPoolId");
267 final long blockId = (Long)m.get("blockId");
268 final long numBytes = (Long)m.get("numBytes");
269 final long generationStamp = (Long)m.get("generationStamp");
270 return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
271 }
272
273 /** Convert a DatanodeInfo to a Json map. */
274 private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
275 if (datanodeinfo == null) {
276 return null;
277 }
278
279 final Map<String, Object> m = new TreeMap<String, Object>();
280 m.put("ipAddr", datanodeinfo.getIpAddr());
281 m.put("hostName", datanodeinfo.getHostName());
282 m.put("storageID", datanodeinfo.getStorageID());
283 m.put("xferPort", datanodeinfo.getXferPort());
284 m.put("infoPort", datanodeinfo.getInfoPort());
285 m.put("infoSecurePort", datanodeinfo.getInfoSecurePort());
286 m.put("ipcPort", datanodeinfo.getIpcPort());
287
288 m.put("capacity", datanodeinfo.getCapacity());
289 m.put("dfsUsed", datanodeinfo.getDfsUsed());
290 m.put("remaining", datanodeinfo.getRemaining());
291 m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
292 m.put("lastUpdate", datanodeinfo.getLastUpdate());
293 m.put("xceiverCount", datanodeinfo.getXceiverCount());
294 m.put("networkLocation", datanodeinfo.getNetworkLocation());
295 m.put("adminState", datanodeinfo.getAdminState().name());
296 return m;
297 }
298
299 /** Convert a Json map to an DatanodeInfo object. */
300 private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
301 if (m == null) {
302 return null;
303 }
304
305 return new DatanodeInfo(
306 (String)m.get("ipAddr"),
307 (String)m.get("hostName"),
308 (String)m.get("storageID"),
309 (int)(long)(Long)m.get("xferPort"),
310 (int)(long)(Long)m.get("infoPort"),
311 (int)(long)(Long)m.get("infoSecurePort"),
312 (int)(long)(Long)m.get("ipcPort"),
313
314 (Long)m.get("capacity"),
315 (Long)m.get("dfsUsed"),
316 (Long)m.get("remaining"),
317 (Long)m.get("blockPoolUsed"),
318 (Long)m.get("lastUpdate"),
319 (int)(long)(Long)m.get("xceiverCount"),
320 (String)m.get("networkLocation"),
321 AdminStates.valueOf((String)m.get("adminState")));
322 }
323
324 /** Convert a DatanodeInfo[] to a Json array. */
325 private static Object[] toJsonArray(final DatanodeInfo[] array) {
326 if (array == null) {
327 return null;
328 } else if (array.length == 0) {
329 return EMPTY_OBJECT_ARRAY;
330 } else {
331 final Object[] a = new Object[array.length];
332 for(int i = 0; i < array.length; i++) {
333 a[i] = toJsonMap(array[i]);
334 }
335 return a;
336 }
337 }
338
339 /** Convert an Object[] to a DatanodeInfo[]. */
340 private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
341 if (objects == null) {
342 return null;
343 } else if (objects.length == 0) {
344 return EMPTY_DATANODE_INFO_ARRAY;
345 } else {
346 final DatanodeInfo[] array = new DatanodeInfo[objects.length];
347 for(int i = 0; i < array.length; i++) {
348 array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
349 }
350 return array;
351 }
352 }
353
354 /** Convert a LocatedBlock to a Json map. */
355 private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
356 ) throws IOException {
357 if (locatedblock == null) {
358 return null;
359 }
360
361 final Map<String, Object> m = new TreeMap<String, Object>();
362 m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
363 m.put("isCorrupt", locatedblock.isCorrupt());
364 m.put("startOffset", locatedblock.getStartOffset());
365 m.put("block", toJsonMap(locatedblock.getBlock()));
366 m.put("locations", toJsonArray(locatedblock.getLocations()));
367 return m;
368 }
369
370 /** Convert a Json map to LocatedBlock. */
371 private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
372 if (m == null) {
373 return null;
374 }
375
376 final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
377 final DatanodeInfo[] locations = toDatanodeInfoArray(
378 (Object[])m.get("locations"));
379 final long startOffset = (Long)m.get("startOffset");
380 final boolean isCorrupt = (Boolean)m.get("isCorrupt");
381
382 final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
383 locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
384 return locatedblock;
385 }
386
387 /** Convert a LocatedBlock[] to a Json array. */
388 private static Object[] toJsonArray(final List<LocatedBlock> array
389 ) throws IOException {
390 if (array == null) {
391 return null;
392 } else if (array.size() == 0) {
393 return EMPTY_OBJECT_ARRAY;
394 } else {
395 final Object[] a = new Object[array.size()];
396 for(int i = 0; i < array.size(); i++) {
397 a[i] = toJsonMap(array.get(i));
398 }
399 return a;
400 }
401 }
402
403 /** Convert an Object[] to a List of LocatedBlock. */
404 private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
405 ) throws IOException {
406 if (objects == null) {
407 return null;
408 } else if (objects.length == 0) {
409 return Collections.emptyList();
410 } else {
411 final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
412 for(int i = 0; i < objects.length; i++) {
413 list.add(toLocatedBlock((Map<?, ?>)objects[i]));
414 }
415 return list;
416 }
417 }
418
419 /** Convert LocatedBlocks to a Json string. */
420 public static String toJsonString(final LocatedBlocks locatedblocks
421 ) throws IOException {
422 if (locatedblocks == null) {
423 return null;
424 }
425
426 final Map<String, Object> m = new TreeMap<String, Object>();
427 m.put("fileLength", locatedblocks.getFileLength());
428 m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
429
430 m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
431 m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
432 m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
433 return toJsonString(LocatedBlocks.class, m);
434 }
435
436 /** Convert a Json map to LocatedBlock. */
437 public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
438 ) throws IOException {
439 if (json == null) {
440 return null;
441 }
442
443 final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
444 final long fileLength = (Long)m.get("fileLength");
445 final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
446 final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
447 (Object[])m.get("locatedBlocks"));
448 final LocatedBlock lastLocatedBlock = toLocatedBlock(
449 (Map<?, ?>)m.get("lastLocatedBlock"));
450 final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
451 return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
452 lastLocatedBlock, isLastBlockComplete);
453 }
454
455 /** Convert a ContentSummary to a Json string. */
456 public static String toJsonString(final ContentSummary contentsummary) {
457 if (contentsummary == null) {
458 return null;
459 }
460
461 final Map<String, Object> m = new TreeMap<String, Object>();
462 m.put("length", contentsummary.getLength());
463 m.put("fileCount", contentsummary.getFileCount());
464 m.put("directoryCount", contentsummary.getDirectoryCount());
465 m.put("quota", contentsummary.getQuota());
466 m.put("spaceConsumed", contentsummary.getSpaceConsumed());
467 m.put("spaceQuota", contentsummary.getSpaceQuota());
468 return toJsonString(ContentSummary.class, m);
469 }
470
471 /** Convert a Json map to a ContentSummary. */
472 public static ContentSummary toContentSummary(final Map<?, ?> json) {
473 if (json == null) {
474 return null;
475 }
476
477 final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
478 final long length = (Long)m.get("length");
479 final long fileCount = (Long)m.get("fileCount");
480 final long directoryCount = (Long)m.get("directoryCount");
481 final long quota = (Long)m.get("quota");
482 final long spaceConsumed = (Long)m.get("spaceConsumed");
483 final long spaceQuota = (Long)m.get("spaceQuota");
484
485 return new ContentSummary(length, fileCount, directoryCount,
486 quota, spaceConsumed, spaceQuota);
487 }
488
489 /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
490 public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
491 if (checksum == null) {
492 return null;
493 }
494
495 final Map<String, Object> m = new TreeMap<String, Object>();
496 m.put("algorithm", checksum.getAlgorithmName());
497 m.put("length", checksum.getLength());
498 m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
499 return toJsonString(FileChecksum.class, m);
500 }
501
502 /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
503 public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
504 final Map<?, ?> json) throws IOException {
505 if (json == null) {
506 return null;
507 }
508
509 final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
510 final String algorithm = (String)m.get("algorithm");
511 final int length = (int)(long)(Long)m.get("length");
512 final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
513
514 final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
515 final DataChecksum.Type crcType =
516 MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
517 final MD5MD5CRC32FileChecksum checksum;
518
519 // Recreate what DFSClient would have returned.
520 switch(crcType) {
521 case CRC32:
522 checksum = new MD5MD5CRC32GzipFileChecksum();
523 break;
524 case CRC32C:
525 checksum = new MD5MD5CRC32CastagnoliFileChecksum();
526 break;
527 default:
528 throw new IOException("Unknown algorithm: " + algorithm);
529 }
530 checksum.readFields(in);
531
532 //check algorithm name
533 if (!checksum.getAlgorithmName().equals(algorithm)) {
534 throw new IOException("Algorithm not matched. Expected " + algorithm
535 + ", Received " + checksum.getAlgorithmName());
536 }
537 //check length
538 if (length != checksum.getLength()) {
539 throw new IOException("Length not matched: length=" + length
540 + ", checksum.getLength()=" + checksum.getLength());
541 }
542
543 return checksum;
544 }
545 }