001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.web;
019
020import java.io.ByteArrayInputStream;
021import java.io.DataInputStream;
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import java.util.Map;
027import java.util.TreeMap;
028
029import org.apache.hadoop.fs.ContentSummary;
030import org.apache.hadoop.fs.FileChecksum;
031import org.apache.hadoop.fs.FileStatus;
032import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
033import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
034import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
035import org.apache.hadoop.fs.permission.FsPermission;
036import org.apache.hadoop.hdfs.DFSUtil;
037import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
038import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
039import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
040import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
041import org.apache.hadoop.hdfs.protocol.LocatedBlock;
042import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
043import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
044import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
045import org.apache.hadoop.io.MD5Hash;
046import org.apache.hadoop.ipc.RemoteException;
047import org.apache.hadoop.security.token.Token;
048import org.apache.hadoop.security.token.TokenIdentifier;
049import org.apache.hadoop.util.DataChecksum;
050import org.apache.hadoop.util.StringUtils;
051import org.mortbay.util.ajax.JSON;
052
053/** JSON Utilities */
054public class JsonUtil {
055  private static final Object[] EMPTY_OBJECT_ARRAY = {};
056  private static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
057
058  /** Convert a token object to a Json string. */
059  public static String toJsonString(final Token<? extends TokenIdentifier> token
060      ) throws IOException {
061    return toJsonString(Token.class, toJsonMap(token));
062  }
063
064  private static Map<String, Object> toJsonMap(
065      final Token<? extends TokenIdentifier> token) throws IOException {
066    if (token == null) {
067      return null;
068    }
069
070    final Map<String, Object> m = new TreeMap<String, Object>();
071    m.put("urlString", token.encodeToUrlString());
072    return m;
073  }
074
075  /** Convert a Json map to a Token. */
076  public static Token<? extends TokenIdentifier> toToken(
077      final Map<?, ?> m) throws IOException {
078    if (m == null) {
079      return null;
080    }
081
082    final Token<DelegationTokenIdentifier> token
083        = new Token<DelegationTokenIdentifier>();
084    token.decodeFromUrlString((String)m.get("urlString"));
085    return token;
086  }
087
088  /** Convert a Json map to a Token of DelegationTokenIdentifier. */
089  @SuppressWarnings("unchecked")
090  public static Token<DelegationTokenIdentifier> toDelegationToken(
091      final Map<?, ?> json) throws IOException {
092    final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName());
093    return (Token<DelegationTokenIdentifier>)toToken(m);
094  }
095
096  /** Convert a Json map to a Token of BlockTokenIdentifier. */
097  @SuppressWarnings("unchecked")
098  private static Token<BlockTokenIdentifier> toBlockToken(
099      final Map<?, ?> m) throws IOException {
100    return (Token<BlockTokenIdentifier>)toToken(m);
101  }
102
103  /** Convert a Token[] to a JSON array. */
104  private static Object[] toJsonArray(final Token<? extends TokenIdentifier>[] array
105      ) throws IOException {
106    if (array == null) {
107      return null;
108    } else if (array.length == 0) {
109      return EMPTY_OBJECT_ARRAY;
110    } else {
111      final Object[] a = new Object[array.length];
112      for(int i = 0; i < array.length; i++) {
113        a[i] = toJsonMap(array[i]);
114      }
115      return a;
116    }
117  }
118
119  /** Convert a token object to a JSON string. */
120  public static String toJsonString(final Token<? extends TokenIdentifier>[] tokens
121      ) throws IOException {
122    if (tokens == null) {
123      return null;
124    }
125
126    final Map<String, Object> m = new TreeMap<String, Object>();
127    m.put(Token.class.getSimpleName(), toJsonArray(tokens));
128    return toJsonString(Token.class.getSimpleName() + "s", m);
129  }
130
131  /** Convert an Object[] to a List<Token<?>>.  */
132  private static List<Token<?>> toTokenList(final Object[] objects) throws IOException {
133    if (objects == null) {
134      return null;
135    } else if (objects.length == 0) {
136      return Collections.emptyList();
137    } else {
138      final List<Token<?>> list = new ArrayList<Token<?>>(objects.length);
139      for(int i = 0; i < objects.length; i++) {
140        list.add(toToken((Map<?, ?>)objects[i]));
141      }
142      return list;
143    }
144  }
145
146  /** Convert a JSON map to a List<Token<?>>. */
147  public static List<Token<?>> toTokenList(final Map<?, ?> json) throws IOException {
148    if (json == null) {
149      return null;
150    }
151
152    final Map<?, ?> m = (Map<?, ?>)json.get(Token.class.getSimpleName() + "s");
153    return toTokenList((Object[])m.get(Token.class.getSimpleName()));
154  }
155
156  /** Convert an exception object to a Json string. */
157  public static String toJsonString(final Exception e) {
158    final Map<String, Object> m = new TreeMap<String, Object>();
159    m.put("exception", e.getClass().getSimpleName());
160    m.put("message", e.getMessage());
161    m.put("javaClassName", e.getClass().getName());
162    return toJsonString(RemoteException.class, m);
163  }
164
165  /** Convert a Json map to a RemoteException. */
166  public static RemoteException toRemoteException(final Map<?, ?> json) {
167    final Map<?, ?> m = (Map<?, ?>)json.get(RemoteException.class.getSimpleName());
168    final String message = (String)m.get("message");
169    final String javaClassName = (String)m.get("javaClassName");
170    return new RemoteException(javaClassName, message);
171  }
172
173  private static String toJsonString(final Class<?> clazz, final Object value) {
174    return toJsonString(clazz.getSimpleName(), value);
175  }
176
177  /** Convert a key-value pair to a Json string. */
178  public static String toJsonString(final String key, final Object value) {
179    final Map<String, Object> m = new TreeMap<String, Object>();
180    m.put(key, value);
181    return JSON.toString(m);
182  }
183
184  /** Convert a FsPermission object to a string. */
185  private static String toString(final FsPermission permission) {
186    return String.format("%o", permission.toShort());
187  }
188
189  /** Convert a string to a FsPermission object. */
190  private static FsPermission toFsPermission(final String s) {
191    return new FsPermission(Short.parseShort(s, 8));
192  }
193
194  static enum PathType {
195    FILE, DIRECTORY, SYMLINK;
196    
197    static PathType valueOf(HdfsFileStatus status) {
198      return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
199    }
200  }
201
202  /** Convert a HdfsFileStatus object to a Json string. */
203  public static String toJsonString(final HdfsFileStatus status,
204      boolean includeType) {
205    if (status == null) {
206      return null;
207    }
208    final Map<String, Object> m = new TreeMap<String, Object>();
209    m.put("pathSuffix", status.getLocalName());
210    m.put("type", PathType.valueOf(status));
211    if (status.isSymlink()) {
212      m.put("symlink", status.getSymlink());
213    }
214
215    m.put("length", status.getLen());
216    m.put("owner", status.getOwner());
217    m.put("group", status.getGroup());
218    m.put("permission", toString(status.getPermission()));
219    m.put("accessTime", status.getAccessTime());
220    m.put("modificationTime", status.getModificationTime());
221    m.put("blockSize", status.getBlockSize());
222    m.put("replication", status.getReplication());
223    return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
224  }
225
226  /** Convert a Json map to a HdfsFileStatus object. */
227  public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
228    if (json == null) {
229      return null;
230    }
231
232    final Map<?, ?> m = includesType ? 
233        (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
234    final String localName = (String) m.get("pathSuffix");
235    final PathType type = PathType.valueOf((String) m.get("type"));
236    final byte[] symlink = type != PathType.SYMLINK? null
237        : DFSUtil.string2Bytes((String)m.get("symlink"));
238
239    final long len = (Long) m.get("length");
240    final String owner = (String) m.get("owner");
241    final String group = (String) m.get("group");
242    final FsPermission permission = toFsPermission((String) m.get("permission"));
243    final long aTime = (Long) m.get("accessTime");
244    final long mTime = (Long) m.get("modificationTime");
245    final long blockSize = (Long) m.get("blockSize");
246    final short replication = (short) (long) (Long) m.get("replication");
247    return new HdfsFileStatus(len, type == PathType.DIRECTORY, replication,
248        blockSize, mTime, aTime, permission, owner, group,
249        symlink, DFSUtil.string2Bytes(localName));
250  }
251
252  /** Convert an ExtendedBlock to a Json map. */
253  private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
254    if (extendedblock == null) {
255      return null;
256    }
257
258    final Map<String, Object> m = new TreeMap<String, Object>();
259    m.put("blockPoolId", extendedblock.getBlockPoolId());
260    m.put("blockId", extendedblock.getBlockId());
261    m.put("numBytes", extendedblock.getNumBytes());
262    m.put("generationStamp", extendedblock.getGenerationStamp());
263    return m;
264  }
265
266  /** Convert a Json map to an ExtendedBlock object. */
267  private static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
268    if (m == null) {
269      return null;
270    }
271    
272    final String blockPoolId = (String)m.get("blockPoolId");
273    final long blockId = (Long)m.get("blockId");
274    final long numBytes = (Long)m.get("numBytes");
275    final long generationStamp = (Long)m.get("generationStamp");
276    return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
277  }
278  
279  /** Convert a DatanodeInfo to a Json map. */
280  private static Map<String, Object> toJsonMap(final DatanodeInfo datanodeinfo) {
281    if (datanodeinfo == null) {
282      return null;
283    }
284
285    final Map<String, Object> m = new TreeMap<String, Object>();
286    m.put("name", datanodeinfo.getName());
287    m.put("storageID", datanodeinfo.getStorageID());
288    m.put("infoPort", datanodeinfo.getInfoPort());
289
290    m.put("ipcPort", datanodeinfo.getIpcPort());
291
292    m.put("capacity", datanodeinfo.getCapacity());
293    m.put("dfsUsed", datanodeinfo.getDfsUsed());
294    m.put("remaining", datanodeinfo.getRemaining());
295    m.put("blockPoolUsed", datanodeinfo.getBlockPoolUsed());
296    m.put("lastUpdate", datanodeinfo.getLastUpdate());
297    m.put("xceiverCount", datanodeinfo.getXceiverCount());
298    m.put("networkLocation", datanodeinfo.getNetworkLocation());
299    m.put("hostName", datanodeinfo.getHostName());
300    m.put("adminState", datanodeinfo.getAdminState().name());
301    return m;
302  }
303
304  /** Convert a Json map to an DatanodeInfo object. */
305  private static DatanodeInfo toDatanodeInfo(final Map<?, ?> m) {
306    if (m == null) {
307      return null;
308    }
309
310    return new DatanodeInfo(
311        (String)m.get("name"),
312        (String)m.get("storageID"),
313        (int)(long)(Long)m.get("infoPort"),
314        (int)(long)(Long)m.get("ipcPort"),
315
316        (Long)m.get("capacity"),
317        (Long)m.get("dfsUsed"),
318        (Long)m.get("remaining"),
319        (Long)m.get("blockPoolUsed"),
320        (Long)m.get("lastUpdate"),
321        (int)(long)(Long)m.get("xceiverCount"),
322        (String)m.get("networkLocation"),
323        (String)m.get("hostName"),
324        AdminStates.valueOf((String)m.get("adminState")));
325  }
326
327  /** Convert a DatanodeInfo[] to a Json array. */
328  private static Object[] toJsonArray(final DatanodeInfo[] array) {
329    if (array == null) {
330      return null;
331    } else if (array.length == 0) {
332      return EMPTY_OBJECT_ARRAY;
333    } else {
334      final Object[] a = new Object[array.length];
335      for(int i = 0; i < array.length; i++) {
336        a[i] = toJsonMap(array[i]);
337      }
338      return a;
339    }
340  }
341
342  /** Convert an Object[] to a DatanodeInfo[]. */
343  private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
344    if (objects == null) {
345      return null;
346    } else if (objects.length == 0) {
347      return EMPTY_DATANODE_INFO_ARRAY;
348    } else {
349      final DatanodeInfo[] array = new DatanodeInfo[objects.length];
350      for(int i = 0; i < array.length; i++) {
351        array[i] = toDatanodeInfo((Map<?, ?>) objects[i]);
352      }
353      return array;
354    }
355  }
356  
357  /** Convert a LocatedBlock to a Json map. */
358  private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
359      ) throws IOException {
360    if (locatedblock == null) {
361      return null;
362    }
363 
364    final Map<String, Object> m = new TreeMap<String, Object>();
365    m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
366    m.put("isCorrupt", locatedblock.isCorrupt());
367    m.put("startOffset", locatedblock.getStartOffset());
368    m.put("block", toJsonMap(locatedblock.getBlock()));
369    m.put("locations", toJsonArray(locatedblock.getLocations()));
370    return m;
371  }
372
373  /** Convert a Json map to LocatedBlock. */
374  private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
375    if (m == null) {
376      return null;
377    }
378
379    final ExtendedBlock b = toExtendedBlock((Map<?, ?>)m.get("block"));
380    final DatanodeInfo[] locations = toDatanodeInfoArray(
381        (Object[])m.get("locations"));
382    final long startOffset = (Long)m.get("startOffset");
383    final boolean isCorrupt = (Boolean)m.get("isCorrupt");
384
385    final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
386    locatedblock.setBlockToken(toBlockToken((Map<?, ?>)m.get("blockToken")));
387    return locatedblock;
388  }
389
390  /** Convert a LocatedBlock[] to a Json array. */
391  private static Object[] toJsonArray(final List<LocatedBlock> array
392      ) throws IOException {
393    if (array == null) {
394      return null;
395    } else if (array.size() == 0) {
396      return EMPTY_OBJECT_ARRAY;
397    } else {
398      final Object[] a = new Object[array.size()];
399      for(int i = 0; i < array.size(); i++) {
400        a[i] = toJsonMap(array.get(i));
401      }
402      return a;
403    }
404  }
405
406  /** Convert an Object[] to a List of LocatedBlock. */
407  private static List<LocatedBlock> toLocatedBlockList(final Object[] objects
408      ) throws IOException {
409    if (objects == null) {
410      return null;
411    } else if (objects.length == 0) {
412      return Collections.emptyList();
413    } else {
414      final List<LocatedBlock> list = new ArrayList<LocatedBlock>(objects.length);
415      for(int i = 0; i < objects.length; i++) {
416        list.add(toLocatedBlock((Map<?, ?>)objects[i]));
417      }
418      return list;
419    }
420  }
421
422  /** Convert LocatedBlocks to a Json string. */
423  public static String toJsonString(final LocatedBlocks locatedblocks
424      ) throws IOException {
425    if (locatedblocks == null) {
426      return null;
427    }
428
429    final Map<String, Object> m = new TreeMap<String, Object>();
430    m.put("fileLength", locatedblocks.getFileLength());
431    m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
432
433    m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
434    m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
435    m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
436    return toJsonString(LocatedBlocks.class, m);
437  }
438
439  /** Convert a Json map to LocatedBlock. */
440  public static LocatedBlocks toLocatedBlocks(final Map<?, ?> json
441      ) throws IOException {
442    if (json == null) {
443      return null;
444    }
445
446    final Map<?, ?> m = (Map<?, ?>)json.get(LocatedBlocks.class.getSimpleName());
447    final long fileLength = (Long)m.get("fileLength");
448    final boolean isUnderConstruction = (Boolean)m.get("isUnderConstruction");
449    final List<LocatedBlock> locatedBlocks = toLocatedBlockList(
450        (Object[])m.get("locatedBlocks"));
451    final LocatedBlock lastLocatedBlock = toLocatedBlock(
452        (Map<?, ?>)m.get("lastLocatedBlock"));
453    final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
454    return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
455        lastLocatedBlock, isLastBlockComplete);
456  }
457
458  /** Convert a ContentSummary to a Json string. */
459  public static String toJsonString(final ContentSummary contentsummary) {
460    if (contentsummary == null) {
461      return null;
462    }
463
464    final Map<String, Object> m = new TreeMap<String, Object>();
465    m.put("length", contentsummary.getLength());
466    m.put("fileCount", contentsummary.getFileCount());
467    m.put("directoryCount", contentsummary.getDirectoryCount());
468    m.put("quota", contentsummary.getQuota());
469    m.put("spaceConsumed", contentsummary.getSpaceConsumed());
470    m.put("spaceQuota", contentsummary.getSpaceQuota());
471    return toJsonString(ContentSummary.class, m);
472  }
473
474  /** Convert a Json map to a ContentSummary. */
475  public static ContentSummary toContentSummary(final Map<?, ?> json) {
476    if (json == null) {
477      return null;
478    }
479
480    final Map<?, ?> m = (Map<?, ?>)json.get(ContentSummary.class.getSimpleName());
481    final long length = (Long)m.get("length");
482    final long fileCount = (Long)m.get("fileCount");
483    final long directoryCount = (Long)m.get("directoryCount");
484    final long quota = (Long)m.get("quota");
485    final long spaceConsumed = (Long)m.get("spaceConsumed");
486    final long spaceQuota = (Long)m.get("spaceQuota");
487
488    return new ContentSummary(length, fileCount, directoryCount,
489        quota, spaceConsumed, spaceQuota);
490  }
491
492  /** Convert a MD5MD5CRC32FileChecksum to a Json string. */
493  public static String toJsonString(final MD5MD5CRC32FileChecksum checksum) {
494    if (checksum == null) {
495      return null;
496    }
497
498    final Map<String, Object> m = new TreeMap<String, Object>();
499    m.put("algorithm", checksum.getAlgorithmName());
500    m.put("length", checksum.getLength());
501    m.put("bytes", StringUtils.byteToHexString(checksum.getBytes()));
502    return toJsonString(FileChecksum.class, m);
503  }
504
505  /** Convert a Json map to a MD5MD5CRC32FileChecksum. */
506  public static MD5MD5CRC32FileChecksum toMD5MD5CRC32FileChecksum(
507      final Map<?, ?> json) throws IOException {
508    if (json == null) {
509      return null;
510    }
511
512    final Map<?, ?> m = (Map<?, ?>)json.get(FileChecksum.class.getSimpleName());
513    final String algorithm = (String)m.get("algorithm");
514    final int length = (int)(long)(Long)m.get("length");
515    final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes"));
516
517    final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
518    final DataChecksum.Type crcType = 
519        MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm);
520    final MD5MD5CRC32FileChecksum checksum;
521
522    // Recreate what DFSClient would have returned.
523    switch(crcType) {
524      case CRC32:
525        checksum = new MD5MD5CRC32GzipFileChecksum();
526        break;
527      case CRC32C:
528        checksum = new MD5MD5CRC32CastagnoliFileChecksum();
529        break;
530      default:
531        throw new IOException("Unknown algorithm: " + algorithm);
532    }
533    checksum.readFields(in);
534
535    //check algorithm name
536    if (!checksum.getAlgorithmName().equals(algorithm)) {
537      throw new IOException("Algorithm not matched. Expected " + algorithm
538          + ", Received " + checksum.getAlgorithmName());
539    }
540    //check length
541    if (length != checksum.getLength()) {
542      throw new IOException("Length not matched: length=" + length
543          + ", checksum.getLength()=" + checksum.getLength());
544    }
545
546    return checksum;
547  }
548}