001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.web;
019
020 import java.io.ByteArrayInputStream;
021 import java.io.DataInputStream;
022 import java.io.IOException;
023 import java.util.ArrayList;
024 import java.util.Collections;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.TreeMap;
028
029 import org.apache.hadoop.fs.ContentSummary;
030 import org.apache.hadoop.fs.FileChecksum;
031 import org.apache.hadoop.fs.FileStatus;
032 import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
033 import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
034 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
035 import org.apache.hadoop.fs.permission.FsPermission;
036 import org.apache.hadoop.hdfs.DFSUtil;
037 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
038 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
039 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
040 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
041 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
042 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
043 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
044 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
045 import org.apache.hadoop.io.MD5Hash;
046 import org.apache.hadoop.ipc.RemoteException;
047 import org.apache.hadoop.security.token.Token;
048 import org.apache.hadoop.security.token.TokenIdentifier;
049 import org.apache.hadoop.util.DataChecksum;
050 import org.apache.hadoop.util.StringUtils;
051 import org.mortbay.util.ajax.JSON;
052
053 /** JSON Utilities */
054 public class JsonUtil {
055 private static final Object[] EMPTY_OBJECT_ARRAY = {};
056 private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
057
058 /** Convert a token object to a Json string. */
059 public static String toJsonString(final Token<? extends TokenIdentifier> token
060 ) throws IOException {
061 return toJsonString(Token.class, toJsonMap(token));
062 }
063
064 private static Map<String, Object> toJsonMap(
065 final Token<? extends TokenIdentifier> token) throws IOException {
066 if (token == null) {
067 return null;
068 }
069
070 final Map<String, Object> m = new TreeMap<String, Object>();
071 m.put("urlString", token.encodeToUrlString());
072 return m;
073 }
074
075 /** Convert a Json map to a Token. */
076 public static Token<? extends TokenIdentifier> toToken(
077 final Map<?, ?> m) throws IOException {
078 if (m == null) {
079 return null;
080 }
081
082 final Token<DelegationTokenIdentifier> token
083 = new Token<DelegationTokenIdentifier>();
084 token.decodeFromUrlString((String)m.get("urlString"));
085 return token;
086 }
087
088 /** Convert a Json map to a Token of DelegationTokenIdentifier. */
089 @SuppressWarnings("unchecked")
090 public static Token<DelegationTokenIdentifier> toDelegationToken(
091 final Map<?, ?> json) throws IOException {
092 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
093 return (Token<DelegationTokenIdentifier>)toToken(m);
094 }
095
096 /** Convert a Json map to a Token of BlockTokenIdentifier. */
097 @SuppressWarnings("unchecked")
098 private static Token<BlockTokenIdentifier> toBlockToken(
099 final Map<?, ?> m) throws IOException {
100 return (Token<BlockTokenIdentifier>)toToken(m);
101 }
102
103 /** Convert a Token[] to a JSON array. */
104 private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
105 ) throws IOException {
106 if (array == null) {
107 return null;
108 } else if (array.length == 0) {
109 return EMPTY_OBJECT_ARRAY;
110 } else {
111 final Object[] a = new Object[array.length];
112 for(int i = 0; i < array.length; i++) {
113 a[i] = toJsonMap(array[i]);
114 }
115 return a;
116 }
117 }
118
119 /** Convert a token object to a JSON string. */
120 public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
121 ) throws IOException {
122 if (tokens == null) {
123 return null;
124 }
125
126 final Map<String, Object> m = new TreeMap<String, Object>();
127 m.put(Token.class.getSimpleName(), toJsonArray(tokens));
128 return toJsonString(Token.class.getSimpleName() + "s", m);
129 }
130
131 /** Convert an Object[] to a List<Token<?>>. */
132 private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
133 if (objects == null) {
134 return null;
135 } else if (objects.length == 0) {
136 return Collections.emptyList();
137 } else {
138 final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
139 for(int i = 0; i < objects.length; i++) {
140 list.add(toToken((Map<?, ?>)objects[i]));
141 }
142 return list;
143 }
144 }
145
146 /** Convert a JSON map to a List<Token<?>>. */
147 public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
148 if (json == null) {
149 return null;
150 }
151
152 final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
153 return toTokenList((Object[])m.get(Token.class.getSimpleName()));
154 }
155
156 /** Convert an exception object to a Json string. */
157 public static String toJsonString(final Exception e) {
158 final Map<String, Object> m = new TreeMap<String, Object>();
159 m.put("exception", e.getClass().getSimpleName());
160 m.put("message", e.getMessage());
161 m.put("javaClassName", e.getClass().getName());
162 return toJsonString(RemoteException.class, m);
163 }
164
165 /** Convert a Json map to a RemoteException. */
166 public static RemoteException toRemoteException(final Map<?, ?> json) {
167 final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
168 final String message = (String)m.get("message");
169 final String javaClassName = (String)m.get("javaClassName");
170 return new RemoteException(javaClassName, message);
171 }
172
173 private static String toJsonString(final Class<?> clazz, final Object value) {
174 return toJsonString(clazz.getSimpleName(), value);
175 }
176
177 /** Convert a key-value pair to a Json string. */
178 public static String toJsonString(final String key, final Object value) {
179 final Map<String, Object> m = new TreeMap<String, Object>();
180 m.put(key, value);
181 return JSON.toString(m);
182 }
183
184 /** Convert a FsPermission object to a string. */
185 private static String toString(final FsPermission permission) {
186 return String.format("%o", permission.toShort());
187 }
188
189 /** Convert a string to a FsPermission object. */
190 private static FsPermission toFsPermission(final String s) {
191 return new FsPermission(Short.parseShort(s, 8));
192 }
193
194 static enum PathType {
195 FILE, DIRECTORY, SYMLINK;
196
197 static PathType valueOf(HdfsFileStatus status) {
198 return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
199 }
200 }
201
202 /** Convert a HdfsFileStatus object to a Json string. */
203 public static String toJsonString(final HdfsFileStatus status,
204 boolean includeType) {
205 if (status == null) {
206 return null;
207 }
208 final Map<String, Object> m = new TreeMap<String, Object>();
209 m.put("pathSuffix", status.getLocalName());
210 m.put("type", PathType.valueOf(status));
211 if (status.isSymlink()) {
212 m.put("symlink", status.getSymlink());
213 }
214
215 m.put("length", status.getLen());
216 m.put("owner", status.getOwner());
217 m.put("group", status.getGroup());
218 m.put("permission", toString(status.getPermission()));
219 m.put("accessTime", status.getAccessTime());
220 m.put("modificationTime", status.getModificationTime());
221 m.put("blockSize", status.getBlockSize());
222 m.put("replication", status.getReplication());
223 return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
224 }
225
226 /** Convert a Json map to a HdfsFileStatus object. */
227 public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
228 if (json == null) {
229 return null;
230 }
231
232 final Map<?, ?> m = includesType ?
233 (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
234 final String localName = (String) m.get("pathSuffix");
235 final PathType type = PathType.valueOf((String) m.get("type"));
236 final byte[] symlink = type != PathType.SYMLINK? null
237 : DFSUtil.string2Bytes((String)m.get("symlink"));
238
239 final long len = (Long) m.get("length");
240 final String owner = (String) m.get("owner");
241 final String group = (String) m.get("group");
242 final FsPermission permission = toFsPermission((String) m.get("permission"));
243 final long aTime = (Long) m.get("accessTime");
244 final long mTime = (Long) m.get("modificationTime");
245 final long blockSize = (Long) m.get("blockSize");
246 final short replication = (short) (long) (Long) m.get("replication");
247 return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
248 blockSize, mTime, aTime, permission, owner, group,
249 symlink, DFSUtil.string2Bytes(localName));
250 }
251
252 /** Convert an ExtendedBlock to a Json map. */
253 private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
254 if (extendedblock == null) {
255 return null;
256 }
257
258 final Map<String, Object> m = new TreeMap<String, Object>();
259 m.put("blockPoolId", extendedblock.getBlockPoolId());
260 m.put("blockId", extendedblock.getBlockId());
261 m.put("numBytes", extendedblock.getNumBytes());
262 m.put("generationStamp", extendedblock.getGenerationStamp());
263 return m;
264 }
265
266 /** Convert a Json map to an ExtendedBlock object. */
267 private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
268 if (m == null) {
269 return null;
270 }
271
272 final String blockPoolId = (String)m.get("blockPoolId");
273 final long blockId = (Long)m.get("blockId");
274 final long numBytes = (Long)m.get("numBytes");
275 final long generationStamp = (Long)m.get("generationStamp");
276 return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
277 }
278
279 /** Convert a DatanodeInfo to a Json map. */
280 private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
281 if (datanodeinfo == null) {
282 return null;
283 }
284
285 final Map<String, Object> m = new TreeMap<String, Object>();
286 m.put("name", datanodeinfo.getName());
287 m.put("storageID", datanodeinfo.getStorageID());
288 m.put("infoPort", datanodeinfo.getInfoPort());
289
290 m.put("ipcPort", datanodeinfo.getIpcPort());
291
292 m.put("capacity", datanodeinfo.getCapacity());
293 m.put("dfsUsed", datanodeinfo.getDfsUsed());
294 m.put("remaining", datanodeinfo.getRemaining());
295 m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
296 m.put("lastUpdate", datanodeinfo.getLastUpdate());
297 m.put("xceiverCount", datanodeinfo.getXceiverCount());
298 m.put("networkLocation", datanodeinfo.getNetworkLocation());
299 m.put("hostName", datanodeinfo.getHostName());
300 m.put("adminState", datanodeinfo.getAdminState().name());
301 return m;
302 }
303
304 /** Convert a Json map to an DatanodeInfo object. */
305 private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
306 if (m == null) {
307 return null;
308 }
309
310 return new DatanodeInfo(
311 (String)m.get("name"),
312 (String)m.get("storageID"),
313 (int)(long)(Long)m.get("infoPort"),
314 (int)(long)(Long)m.get("ipcPort"),
315
316 (Long)m.get("capacity"),
317 (Long)m.get("dfsUsed"),
318 (Long)m.get("remaining"),
319 (Long)m.get("blockPoolUsed"),
320 (Long)m.get("lastUpdate"),
321 (int)(long)(Long)m.get("xceiverCount"),
322 (String)m.get("networkLocation"),
323 (String)m.get("hostName"),
324 AdminStates.valueOf((String)m.get("adminState")));
325 }
326
327 /** Convert a DatanodeInfo[] to a Json array. */
328 private static Object[] toJsonArray(final DatanodeInfo[] array) {
329 if (array == null) {
330 return null;
331 } else if (array.length == 0) {
332 return EMPTY_OBJECT_ARRAY;
333 } else {
334 final Object[] a = new Object[array.length];
335 for(int i = 0; i < array.length; i++) {
336 a[i] = toJsonMap(array[i]);
337 }
338 return a;
339 }
340 }
341
342 /** Convert an Object[] to a DatanodeInfo[]. */
343 private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
344 if (objects == null) {
345 return null;
346 } else if (objects.length == 0) {
347 return EMPTY_DATANODE_INFO_ARRAY;
348 } else {
349 final DatanodeInfo[] array = new DatanodeInfo[objects.length];
350 for(int i = 0; i < array.length; i++) {
351 array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
352 }
353 return array;
354 }
355 }
356
357 /** Convert a LocatedBlock to a Json map. */
358 private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
359 ) throws IOException {
360 if (locatedblock == null) {
361 return null;
362 }
363
364 final Map<String, Object> m = new TreeMap<String, Object>();
365 m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
366 m.put("isCorrupt", locatedblock.isCorrupt());
367 m.put("startOffset", locatedblock.getStartOffset());
368 m.put("block", toJsonMap(locatedblock.getBlock()));
369 m.put("locations", toJsonArray(locatedblock.getLocations()));
370 return m;
371 }
372
373 /** Convert a Json map to LocatedBlock. */
374 private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
375 if (m == null) {
376 return null;
377 }
378
379 final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
380 final DatanodeInfo[] locations = toDatanodeInfoArray(
381 (Object[])m.get("locations"));
382 final long startOffset = (Long)m.get("startOffset");
383 final boolean isCorrupt = (Boolean)m.get("isCorrupt");
384
385 final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
386 locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
387 return locatedblock;
388 }
389
390 /** Convert a LocatedBlock[] to a Json array. */
391 private static Object[] toJsonArray(final List<LocatedBlock> array
392 ) throws IOException {
393 if (array == null) {
394 return null;
395 } else if (array.size() == 0) {
396 return EMPTY_OBJECT_ARRAY;
397 } else {
398 final Object[] a = new Object[array.size()];
399 for(int i = 0; i < array.size(); i++) {
400 a[i] = toJsonMap(array.get(i));
401 }
402 return a;
403 }
404 }
405
406 /** Convert an Object[] to a List of LocatedBlock. */
407 private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
408 ) throws IOException {
409 if (objects == null) {
410 return null;
411 } else if (objects.length == 0) {
412 return Collections.emptyList();
413 } else {
414 final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
415 for(int i = 0; i < objects.length; i++) {
416 list.add(toLocatedBlock((Map<?, ?>)objects[i]));
417 }
418 return list;
419 }
420 }
421
422 /** Convert LocatedBlocks to a Json string. */
423 public static String toJsonString(final LocatedBlocks locatedblocks
424 ) throws IOException {
425 if (locatedblocks == null) {
426 return null;
427 }
428
429 final Map<String, Object> m = new TreeMap<String, Object>();
430 m.put("fileLength", locatedblocks.getFileLength());
431 m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
432
433 m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
434 m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
435 m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
436 return toJsonString(LocatedBlocks.class, m);
437 }
438
439 /** Convert a Json map to LocatedBlock. */
440 public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
441 ) throws IOException {
442 if (json == null) {
443 return null;
444 }
445
446 final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
447 final long fileLength = (Long)m.get("fileLength");
448 final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
449 final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
450 (Object[])m.get("locatedBlocks"));
451 final LocatedBlock lastLocatedBlock = toLocatedBlock(
452 (Map<?, ?>)m.get("lastLocatedBlock"));
453 final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
454 return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
455 lastLocatedBlock, isLastBlockComplete);
456 }
457
458 /** Convert a ContentSummary to a Json string. */
459 public static String toJsonString(final ContentSummary contentsummary) {
460 if (contentsummary == null) {
461 return null;
462 }
463
464 final Map<String, Object> m = new TreeMap<String, Object>();
465 m.put("length", contentsummary.getLength());
466 m.put("fileCount", contentsummary.getFileCount());
467 m.put("directoryCount", contentsummary.getDirectoryCount());
468 m.put("quota", contentsummary.getQuota());
469 m.put("spaceConsumed", contentsummary.getSpaceConsumed());
470 m.put("spaceQuota", contentsummary.getSpaceQuota());
471 return toJsonString(ContentSummary.class, m);
472 }
473
474 /** Convert a Json map to a ContentSummary. */
475 public static ContentSummary toContentSummary(final Map<?, ?> json) {
476 if (json == null) {
477 return null;
478 }
479
480 final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
481 final long length = (Long)m.get("length");
482 final long fileCount = (Long)m.get("fileCount");
483 final long directoryCount = (Long)m.get("directoryCount");
484 final long quota = (Long)m.get("quota");
485 final long spaceConsumed = (Long)m.get("spaceConsumed");
486 final long spaceQuota = (Long)m.get("spaceQuota");
487
488 return new ContentSummary(length, fileCount, directoryCount,
489 quota, spaceConsumed, spaceQuota);
490 }
491
492 /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
493 public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
494 if (checksum == null) {
495 return null;
496 }
497
498 final Map<String, Object> m = new TreeMap<String, Object>();
499 m.put("algorithm", checksum.getAlgorithmName());
500 m.put("length", checksum.getLength());
501 m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
502 return toJsonString(FileChecksum.class, m);
503 }
504
505 /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
506 public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
507 final Map<?, ?> json) throws IOException {
508 if (json == null) {
509 return null;
510 }
511
512 final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
513 final String algorithm = (String)m.get("algorithm");
514 final int length = (int)(long)(Long)m.get("length");
515 final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
516
517 final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
518 final DataChecksum.Type crcType =
519 MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
520 final MD5MD5CRC32FileChecksum checksum;
521
522 // Recreate what DFSClient would have returned.
523 switch(crcType) {
524 case CRC32:
525 checksum = new MD5MD5CRC32GzipFileChecksum();
526 break;
527 case CRC32C:
528 checksum = new MD5MD5CRC32CastagnoliFileChecksum();
529 break;
530 default:
531 throw new IOException("Unknown algorithm: " + algorithm);
532 }
533 checksum.readFields(in);
534
535 //check algorithm name
536 if (!checksum.getAlgorithmName().equals(algorithm)) {
537 throw new IOException("Algorithm not matched. Expected " + algorithm
538 + ", Received " + checksum.getAlgorithmName());
539 }
540 //check length
541 if (length != checksum.getLength()) {
542 throw new IOException("Length not matched: length=" + length
543 + ", checksum.getLength()=" + checksum.getLength());
544 }
545
546 return checksum;
547 }
548 }