001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.crypto.key; 020 021import com.google.common.base.Preconditions; 022import org.apache.commons.io.IOUtils; 023import org.apache.hadoop.classification.InterfaceAudience; 024import org.apache.hadoop.conf.Configuration; 025import org.apache.hadoop.fs.FSDataInputStream; 026import org.apache.hadoop.fs.FSDataOutputStream; 027import org.apache.hadoop.fs.FileStatus; 028import org.apache.hadoop.fs.FileSystem; 029import org.apache.hadoop.fs.Path; 030import org.apache.hadoop.fs.permission.FsPermission; 031import org.apache.hadoop.security.ProviderUtils; 032import org.apache.hadoop.util.StringUtils; 033import org.slf4j.Logger; 034import org.slf4j.LoggerFactory; 035 036import com.google.common.annotations.VisibleForTesting; 037 038import javax.crypto.spec.SecretKeySpec; 039 040import java.io.IOException; 041import java.io.InputStream; 042import java.io.ObjectInputStream; 043import java.io.ObjectOutputStream; 044import java.io.Serializable; 045import java.net.URI; 046import java.net.URL; 047import java.security.Key; 048import java.security.KeyStore; 049import java.security.KeyStoreException; 050import java.security.NoSuchAlgorithmException; 051import java.security.UnrecoverableKeyException; 052import java.security.cert.CertificateException; 053import java.util.ArrayList; 054import java.util.Date; 055import java.util.Enumeration; 056import java.util.HashMap; 057import java.util.List; 058import java.util.Map; 059import java.util.concurrent.locks.Lock; 060import java.util.concurrent.locks.ReadWriteLock; 061import java.util.concurrent.locks.ReentrantReadWriteLock; 062 063/** 064 * KeyProvider based on Java's KeyStore file format. The file may be stored in 065 * any Hadoop FileSystem using the following name mangling: 066 * jks://[email protected]/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks 067 * jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks 068 * <p/> 069 * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set, 070 * its value is used as the password for the keystore. 071 * <p/> 072 * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set, 073 * the password for the keystore is read from file specified in the 074 * {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file 075 * is looked up in Hadoop's configuration directory via the classpath. 076 * <p/> 077 * <b>NOTE:</b> Make sure the password in the password file does not have an 078 * ENTER at the end, else it won't be valid for the Java KeyStore. 079 * <p/> 080 * If the environment variable, nor the property are not set, the password used 081 * is 'none'. 082 * <p/> 083 * It is expected for encrypted InputFormats and OutputFormats to copy the keys 084 * from the original provider into the job's Credentials object, which is 085 * accessed via the UserProvider. Therefore, this provider won't be used by 086 * MapReduce tasks. 087 */ 088@InterfaceAudience.Private 089public class JavaKeyStoreProvider extends KeyProvider { 090 private static final String KEY_METADATA = "KeyMetadata"; 091 private static Logger LOG = 092 LoggerFactory.getLogger(JavaKeyStoreProvider.class); 093 094 public static final String SCHEME_NAME = "jceks"; 095 096 public static final String KEYSTORE_PASSWORD_FILE_KEY = 097 "hadoop.security.keystore.java-keystore-provider.password-file"; 098 099 public static final String KEYSTORE_PASSWORD_ENV_VAR = 100 "HADOOP_KEYSTORE_PASSWORD"; 101 public static final char[] KEYSTORE_PASSWORD_DEFAULT = "none".toCharArray(); 102 103 private final URI uri; 104 private final Path path; 105 private final FileSystem fs; 106 private final FsPermission permissions; 107 private final KeyStore keyStore; 108 private char[] password; 109 private boolean changed = false; 110 private Lock readLock; 111 private Lock writeLock; 112 113 private final Map<String, Metadata> cache = new HashMap<String, Metadata>(); 114 115 @VisibleForTesting 116 JavaKeyStoreProvider(JavaKeyStoreProvider other) { 117 super(new Configuration()); 118 uri = other.uri; 119 path = other.path; 120 fs = other.fs; 121 permissions = other.permissions; 122 keyStore = other.keyStore; 123 password = other.password; 124 changed = other.changed; 125 readLock = other.readLock; 126 writeLock = other.writeLock; 127 } 128 129 private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException { 130 super(conf); 131 this.uri = uri; 132 path = ProviderUtils.unnestUri(uri); 133 fs = path.getFileSystem(conf); 134 // Get the password file from the conf, if not present from the user's 135 // environment var 136 if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) { 137 password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray(); 138 } 139 if (password == null) { 140 String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY); 141 if (pwFile != null) { 142 ClassLoader cl = Thread.currentThread().getContextClassLoader(); 143 URL pwdFile = cl.getResource(pwFile); 144 if (pwdFile == null) { 145 // Provided Password file does not exist 146 throw new IOException("Password file does not exists"); 147 } 148 try (InputStream is = pwdFile.openStream()) { 149 password = IOUtils.toString(is).trim().toCharArray(); 150 } 151 } 152 } 153 if (password == null) { 154 password = KEYSTORE_PASSWORD_DEFAULT; 155 } 156 try { 157 Path oldPath = constructOldPath(path); 158 Path newPath = constructNewPath(path); 159 keyStore = KeyStore.getInstance(SCHEME_NAME); 160 FsPermission perm = null; 161 if (fs.exists(path)) { 162 // flush did not proceed to completion 163 // _NEW should not exist 164 if (fs.exists(newPath)) { 165 throw new IOException( 166 String.format("Keystore not loaded due to some inconsistency " 167 + "('%s' and '%s' should not exist together)!!", path, newPath)); 168 } 169 perm = tryLoadFromPath(path, oldPath); 170 } else { 171 perm = tryLoadIncompleteFlush(oldPath, newPath); 172 } 173 // Need to save off permissions in case we need to 174 // rewrite the keystore in flush() 175 permissions = perm; 176 } catch (KeyStoreException e) { 177 throw new IOException("Can't create keystore", e); 178 } catch (NoSuchAlgorithmException e) { 179 throw new IOException("Can't load keystore " + path, e); 180 } catch (CertificateException e) { 181 throw new IOException("Can't load keystore " + path, e); 182 } 183 ReadWriteLock lock = new ReentrantReadWriteLock(true); 184 readLock = lock.readLock(); 185 writeLock = lock.writeLock(); 186 } 187 188 /** 189 * Try loading from the user specified path, else load from the backup 190 * path in case Exception is not due to bad/wrong password 191 * @param path Actual path to load from 192 * @param backupPath Backup path (_OLD) 193 * @return The permissions of the loaded file 194 * @throws NoSuchAlgorithmException 195 * @throws CertificateException 196 * @throws IOException 197 */ 198 private FsPermission tryLoadFromPath(Path path, Path backupPath) 199 throws NoSuchAlgorithmException, CertificateException, 200 IOException { 201 FsPermission perm = null; 202 try { 203 perm = loadFromPath(path, password); 204 // Remove _OLD if exists 205 if (fs.exists(backupPath)) { 206 fs.delete(backupPath, true); 207 } 208 LOG.debug("KeyStore loaded successfully !!"); 209 } catch (IOException ioe) { 210 // If file is corrupted for some reason other than 211 // wrong password try the _OLD file if exits 212 if (!isBadorWrongPassword(ioe)) { 213 perm = loadFromPath(backupPath, password); 214 // Rename CURRENT to CORRUPTED 215 renameOrFail(path, new Path(path.toString() + "_CORRUPTED_" 216 + System.currentTimeMillis())); 217 renameOrFail(backupPath, path); 218 LOG.debug(String.format( 219 "KeyStore loaded successfully from '%s' since '%s'" 220 + "was corrupted !!", backupPath, path)); 221 } else { 222 throw ioe; 223 } 224 } 225 return perm; 226 } 227 228 /** 229 * The KeyStore might have gone down during a flush, In which case either the 230 * _NEW or _OLD files might exists. This method tries to load the KeyStore 231 * from one of these intermediate files. 232 * @param oldPath the _OLD file created during flush 233 * @param newPath the _NEW file created during flush 234 * @return The permissions of the loaded file 235 * @throws IOException 236 * @throws NoSuchAlgorithmException 237 * @throws CertificateException 238 */ 239 private FsPermission tryLoadIncompleteFlush(Path oldPath, Path newPath) 240 throws IOException, NoSuchAlgorithmException, CertificateException { 241 FsPermission perm = null; 242 // Check if _NEW exists (in case flush had finished writing but not 243 // completed the re-naming) 244 if (fs.exists(newPath)) { 245 perm = loadAndReturnPerm(newPath, oldPath); 246 } 247 // try loading from _OLD (An earlier Flushing MIGHT not have completed 248 // writing completely) 249 if ((perm == null) && fs.exists(oldPath)) { 250 perm = loadAndReturnPerm(oldPath, newPath); 251 } 252 // If not loaded yet, 253 // required to create an empty keystore. *sigh* 254 if (perm == null) { 255 keyStore.load(null, password); 256 LOG.debug("KeyStore initialized anew successfully !!"); 257 perm = new FsPermission("700"); 258 } 259 return perm; 260 } 261 262 private FsPermission loadAndReturnPerm(Path pathToLoad, Path pathToDelete) 263 throws NoSuchAlgorithmException, CertificateException, 264 IOException { 265 FsPermission perm = null; 266 try { 267 perm = loadFromPath(pathToLoad, password); 268 renameOrFail(pathToLoad, path); 269 LOG.debug(String.format("KeyStore loaded successfully from '%s'!!", 270 pathToLoad)); 271 if (fs.exists(pathToDelete)) { 272 fs.delete(pathToDelete, true); 273 } 274 } catch (IOException e) { 275 // Check for password issue : don't want to trash file due 276 // to wrong password 277 if (isBadorWrongPassword(e)) { 278 throw e; 279 } 280 } 281 return perm; 282 } 283 284 private boolean isBadorWrongPassword(IOException ioe) { 285 // As per documentation this is supposed to be the way to figure 286 // if password was correct 287 if (ioe.getCause() instanceof UnrecoverableKeyException) { 288 return true; 289 } 290 // Unfortunately that doesn't seem to work.. 291 // Workaround : 292 if ((ioe.getCause() == null) 293 && (ioe.getMessage() != null) 294 && ((ioe.getMessage().contains("Keystore was tampered")) || (ioe 295 .getMessage().contains("password was incorrect")))) { 296 return true; 297 } 298 return false; 299 } 300 301 private FsPermission loadFromPath(Path p, char[] password) 302 throws IOException, NoSuchAlgorithmException, CertificateException { 303 try (FSDataInputStream in = fs.open(p)) { 304 FileStatus s = fs.getFileStatus(p); 305 keyStore.load(in, password); 306 return s.getPermission(); 307 } 308 } 309 310 private Path constructNewPath(Path path) { 311 Path newPath = new Path(path.toString() + "_NEW"); 312 return newPath; 313 } 314 315 private Path constructOldPath(Path path) { 316 Path oldPath = new Path(path.toString() + "_OLD"); 317 return oldPath; 318 } 319 320 @Override 321 public KeyVersion getKeyVersion(String versionName) throws IOException { 322 readLock.lock(); 323 try { 324 SecretKeySpec key = null; 325 try { 326 if (!keyStore.containsAlias(versionName)) { 327 return null; 328 } 329 key = (SecretKeySpec) keyStore.getKey(versionName, password); 330 } catch (KeyStoreException e) { 331 throw new IOException("Can't get key " + versionName + " from " + 332 path, e); 333 } catch (NoSuchAlgorithmException e) { 334 throw new IOException("Can't get algorithm for key " + key + " from " + 335 path, e); 336 } catch (UnrecoverableKeyException e) { 337 throw new IOException("Can't recover key " + key + " from " + path, e); 338 } 339 return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); 340 } finally { 341 readLock.unlock(); 342 } 343 } 344 345 @Override 346 public List<String> getKeys() throws IOException { 347 readLock.lock(); 348 try { 349 ArrayList<String> list = new ArrayList<String>(); 350 String alias = null; 351 try { 352 Enumeration<String> e = keyStore.aliases(); 353 while (e.hasMoreElements()) { 354 alias = e.nextElement(); 355 // only include the metadata key names in the list of names 356 if (!alias.contains("@")) { 357 list.add(alias); 358 } 359 } 360 } catch (KeyStoreException e) { 361 throw new IOException("Can't get key " + alias + " from " + path, e); 362 } 363 return list; 364 } finally { 365 readLock.unlock(); 366 } 367 } 368 369 @Override 370 public List<KeyVersion> getKeyVersions(String name) throws IOException { 371 readLock.lock(); 372 try { 373 List<KeyVersion> list = new ArrayList<KeyVersion>(); 374 Metadata km = getMetadata(name); 375 if (km != null) { 376 int latestVersion = km.getVersions(); 377 KeyVersion v = null; 378 String versionName = null; 379 for (int i = 0; i < latestVersion; i++) { 380 versionName = buildVersionName(name, i); 381 v = getKeyVersion(versionName); 382 if (v != null) { 383 list.add(v); 384 } 385 } 386 } 387 return list; 388 } finally { 389 readLock.unlock(); 390 } 391 } 392 393 @Override 394 public Metadata getMetadata(String name) throws IOException { 395 readLock.lock(); 396 try { 397 if (cache.containsKey(name)) { 398 return cache.get(name); 399 } 400 try { 401 if (!keyStore.containsAlias(name)) { 402 return null; 403 } 404 Metadata meta = ((KeyMetadata) keyStore.getKey(name, password)).metadata; 405 cache.put(name, meta); 406 return meta; 407 } catch (ClassCastException e) { 408 throw new IOException("Can't cast key for " + name + " in keystore " + 409 path + " to a KeyMetadata. Key may have been added using " + 410 " keytool or some other non-Hadoop method.", e); 411 } catch (KeyStoreException e) { 412 throw new IOException("Can't get metadata for " + name + 413 " from keystore " + path, e); 414 } catch (NoSuchAlgorithmException e) { 415 throw new IOException("Can't get algorithm for " + name + 416 " from keystore " + path, e); 417 } catch (UnrecoverableKeyException e) { 418 throw new IOException("Can't recover key for " + name + 419 " from keystore " + path, e); 420 } 421 } finally { 422 readLock.unlock(); 423 } 424 } 425 426 @Override 427 public KeyVersion createKey(String name, byte[] material, 428 Options options) throws IOException { 429 Preconditions.checkArgument(name.equals(StringUtils.toLowerCase(name)), 430 "Uppercase key names are unsupported: %s", name); 431 writeLock.lock(); 432 try { 433 try { 434 if (keyStore.containsAlias(name) || cache.containsKey(name)) { 435 throw new IOException("Key " + name + " already exists in " + this); 436 } 437 } catch (KeyStoreException e) { 438 throw new IOException("Problem looking up key " + name + " in " + this, 439 e); 440 } 441 Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), 442 options.getDescription(), options.getAttributes(), new Date(), 1); 443 if (options.getBitLength() != 8 * material.length) { 444 throw new IOException("Wrong key length. Required " + 445 options.getBitLength() + ", but got " + (8 * material.length)); 446 } 447 cache.put(name, meta); 448 String versionName = buildVersionName(name, 0); 449 return innerSetKeyVersion(name, versionName, material, meta.getCipher()); 450 } finally { 451 writeLock.unlock(); 452 } 453 } 454 455 @Override 456 public void deleteKey(String name) throws IOException { 457 writeLock.lock(); 458 try { 459 Metadata meta = getMetadata(name); 460 if (meta == null) { 461 throw new IOException("Key " + name + " does not exist in " + this); 462 } 463 for(int v=0; v < meta.getVersions(); ++v) { 464 String versionName = buildVersionName(name, v); 465 try { 466 if (keyStore.containsAlias(versionName)) { 467 keyStore.deleteEntry(versionName); 468 } 469 } catch (KeyStoreException e) { 470 throw new IOException("Problem removing " + versionName + " from " + 471 this, e); 472 } 473 } 474 try { 475 if (keyStore.containsAlias(name)) { 476 keyStore.deleteEntry(name); 477 } 478 } catch (KeyStoreException e) { 479 throw new IOException("Problem removing " + name + " from " + this, e); 480 } 481 cache.remove(name); 482 changed = true; 483 } finally { 484 writeLock.unlock(); 485 } 486 } 487 488 KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, 489 String cipher) throws IOException { 490 try { 491 keyStore.setKeyEntry(versionName, new SecretKeySpec(material, cipher), 492 password, null); 493 } catch (KeyStoreException e) { 494 throw new IOException("Can't store key " + versionName + " in " + this, 495 e); 496 } 497 changed = true; 498 return new KeyVersion(name, versionName, material); 499 } 500 501 @Override 502 public KeyVersion rollNewVersion(String name, 503 byte[] material) throws IOException { 504 writeLock.lock(); 505 try { 506 Metadata meta = getMetadata(name); 507 if (meta == null) { 508 throw new IOException("Key " + name + " not found"); 509 } 510 if (meta.getBitLength() != 8 * material.length) { 511 throw new IOException("Wrong key length. Required " + 512 meta.getBitLength() + ", but got " + (8 * material.length)); 513 } 514 int nextVersion = meta.addVersion(); 515 String versionName = buildVersionName(name, nextVersion); 516 return innerSetKeyVersion(name, versionName, material, meta.getCipher()); 517 } finally { 518 writeLock.unlock(); 519 } 520 } 521 522 @Override 523 public void flush() throws IOException { 524 Path newPath = constructNewPath(path); 525 Path oldPath = constructOldPath(path); 526 Path resetPath = path; 527 writeLock.lock(); 528 try { 529 if (!changed) { 530 return; 531 } 532 // Might exist if a backup has been restored etc. 533 if (fs.exists(newPath)) { 534 renameOrFail(newPath, new Path(newPath.toString() 535 + "_ORPHANED_" + System.currentTimeMillis())); 536 } 537 if (fs.exists(oldPath)) { 538 renameOrFail(oldPath, new Path(oldPath.toString() 539 + "_ORPHANED_" + System.currentTimeMillis())); 540 } 541 // put all of the updates into the keystore 542 for(Map.Entry<String, Metadata> entry: cache.entrySet()) { 543 try { 544 keyStore.setKeyEntry(entry.getKey(), new KeyMetadata(entry.getValue()), 545 password, null); 546 } catch (KeyStoreException e) { 547 throw new IOException("Can't set metadata key " + entry.getKey(),e ); 548 } 549 } 550 551 // Save old File first 552 boolean fileExisted = backupToOld(oldPath); 553 if (fileExisted) { 554 resetPath = oldPath; 555 } 556 // write out the keystore 557 // Write to _NEW path first : 558 try { 559 writeToNew(newPath); 560 } catch (IOException ioe) { 561 // rename _OLD back to curent and throw Exception 562 revertFromOld(oldPath, fileExisted); 563 resetPath = path; 564 throw ioe; 565 } 566 // Rename _NEW to CURRENT and delete _OLD 567 cleanupNewAndOld(newPath, oldPath); 568 changed = false; 569 } catch (IOException ioe) { 570 resetKeyStoreState(resetPath); 571 throw ioe; 572 } finally { 573 writeLock.unlock(); 574 } 575 } 576 577 private void resetKeyStoreState(Path path) { 578 LOG.debug("Could not flush Keystore.." 579 + "attempting to reset to previous state !!"); 580 // 1) flush cache 581 cache.clear(); 582 // 2) load keyStore from previous path 583 try { 584 loadFromPath(path, password); 585 LOG.debug("KeyStore resetting to previously flushed state !!"); 586 } catch (Exception e) { 587 LOG.debug("Could not reset Keystore to previous state", e); 588 } 589 } 590 591 private void cleanupNewAndOld(Path newPath, Path oldPath) throws IOException { 592 // Rename _NEW to CURRENT 593 renameOrFail(newPath, path); 594 // Delete _OLD 595 if (fs.exists(oldPath)) { 596 fs.delete(oldPath, true); 597 } 598 } 599 600 protected void writeToNew(Path newPath) throws IOException { 601 try (FSDataOutputStream out = 602 FileSystem.create(fs, newPath, permissions);) { 603 keyStore.store(out, password); 604 } catch (KeyStoreException e) { 605 throw new IOException("Can't store keystore " + this, e); 606 } catch (NoSuchAlgorithmException e) { 607 throw new IOException( 608 "No such algorithm storing keystore " + this, e); 609 } catch (CertificateException e) { 610 throw new IOException( 611 "Certificate exception storing keystore " + this, e); 612 } 613 } 614 615 protected boolean backupToOld(Path oldPath) 616 throws IOException { 617 boolean fileExisted = false; 618 if (fs.exists(path)) { 619 renameOrFail(path, oldPath); 620 fileExisted = true; 621 } 622 return fileExisted; 623 } 624 625 private void revertFromOld(Path oldPath, boolean fileExisted) 626 throws IOException { 627 if (fileExisted) { 628 renameOrFail(oldPath, path); 629 } 630 } 631 632 633 private void renameOrFail(Path src, Path dest) 634 throws IOException { 635 if (!fs.rename(src, dest)) { 636 throw new IOException("Rename unsuccessful : " 637 + String.format("'%s' to '%s'", src, dest)); 638 } 639 } 640 641 @Override 642 public String toString() { 643 return uri.toString(); 644 } 645 646 /** 647 * The factory to create JksProviders, which is used by the ServiceLoader. 648 */ 649 public static class Factory extends KeyProviderFactory { 650 @Override 651 public KeyProvider createProvider(URI providerName, 652 Configuration conf) throws IOException { 653 if (SCHEME_NAME.equals(providerName.getScheme())) { 654 return new JavaKeyStoreProvider(providerName, conf); 655 } 656 return null; 657 } 658 } 659 660 /** 661 * An adapter between a KeyStore Key and our Metadata. This is used to store 662 * the metadata in a KeyStore even though isn't really a key. 663 */ 664 public static class KeyMetadata implements Key, Serializable { 665 private Metadata metadata; 666 private final static long serialVersionUID = 8405872419967874451L; 667 668 private KeyMetadata(Metadata meta) { 669 this.metadata = meta; 670 } 671 672 @Override 673 public String getAlgorithm() { 674 return metadata.getCipher(); 675 } 676 677 @Override 678 public String getFormat() { 679 return KEY_METADATA; 680 } 681 682 @Override 683 public byte[] getEncoded() { 684 return new byte[0]; 685 } 686 687 private void writeObject(ObjectOutputStream out) throws IOException { 688 byte[] serialized = metadata.serialize(); 689 out.writeInt(serialized.length); 690 out.write(serialized); 691 } 692 693 private void readObject(ObjectInputStream in 694 ) throws IOException, ClassNotFoundException { 695 byte[] buf = new byte[in.readInt()]; 696 in.readFully(buf); 697 metadata = new Metadata(buf); 698 } 699 700 } 701}