001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode;
019
020import java.io.File;
021import java.io.FilenameFilter;
022import java.io.IOException;
023import java.nio.file.Files;
024import java.util.List;
025
026import org.apache.commons.logging.Log;
027import org.apache.commons.logging.LogFactory;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hdfs.server.common.Storage;
030import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
031import org.apache.hadoop.hdfs.server.common.StorageInfo;
032
033import com.google.common.base.Preconditions;
034import org.apache.hadoop.io.IOUtils;
035
036public abstract class NNUpgradeUtil {
037  
038  private static final Log LOG = LogFactory.getLog(NNUpgradeUtil.class);
039  
040  /**
041   * Return true if this storage dir can roll back to the previous storage
042   * state, false otherwise. The NN will refuse to run the rollback operation
043   * unless at least one JM or fsimage storage directory can roll back.
044   * 
045   * @param storage the storage info for the current state
046   * @param prevStorage the storage info for the previous (unupgraded) state
047   * @param targetLayoutVersion the layout version we intend to roll back to
048   * @return true if this JM can roll back, false otherwise.
049   * @throws IOException in the event of error
050   */
051  static boolean canRollBack(StorageDirectory sd, StorageInfo storage,
052      StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
053    File prevDir = sd.getPreviousDir();
054    if (!prevDir.exists()) {  // use current directory then
055      LOG.info("Storage directory " + sd.getRoot()
056               + " does not contain previous fs state.");
057      // read and verify consistency with other directories
058      storage.readProperties(sd);
059      return false;
060    }
061
062    // read and verify consistency of the prev dir
063    prevStorage.readPreviousVersionProperties(sd);
064
065    if (prevStorage.getLayoutVersion() != targetLayoutVersion) {
066      throw new IOException(
067        "Cannot rollback to storage version " +
068        prevStorage.getLayoutVersion() +
069        " using this version of the NameNode, which uses storage version " +
070        targetLayoutVersion + ". " +
071        "Please use the previous version of HDFS to perform the rollback.");
072    }
073    
074    return true;
075  }
076
077  /**
078   * Finalize the upgrade. The previous dir, if any, will be renamed and
079   * removed. After this is completed, rollback is no longer allowed.
080   * 
081   * @param sd the storage directory to finalize
082   * @throws IOException in the event of error
083   */
084  static void doFinalize(StorageDirectory sd) throws IOException {
085    File prevDir = sd.getPreviousDir();
086    if (!prevDir.exists()) { // already discarded
087      LOG.info("Directory " + prevDir + " does not exist.");
088      LOG.info("Finalize upgrade for " + sd.getRoot()+ " is not required.");
089      return;
090    }
091    LOG.info("Finalizing upgrade of storage directory " + sd.getRoot());
092    Preconditions.checkState(sd.getCurrentDir().exists(),
093        "Current directory must exist.");
094    final File tmpDir = sd.getFinalizedTmp();
095    // rename previous to tmp and remove
096    NNStorage.rename(prevDir, tmpDir);
097    NNStorage.deleteDir(tmpDir);
098    LOG.info("Finalize upgrade for " + sd.getRoot()+ " is complete.");
099  }
100  
101  /**
102   * Perform any steps that must succeed across all storage dirs/JournalManagers
103   * involved in an upgrade before proceeding onto the actual upgrade stage. If
104   * a call to any JM's or local storage dir's doPreUpgrade method fails, then
105   * doUpgrade will not be called for any JM. The existing current dir is
106   * renamed to previous.tmp, and then a new, empty current dir is created.
107   *
108   * @param conf configuration for creating {@link EditLogFileOutputStream}
109   * @param sd the storage directory to perform the pre-upgrade procedure.
110   * @throws IOException in the event of error
111   */
112  static void doPreUpgrade(Configuration conf, StorageDirectory sd)
113      throws IOException {
114    LOG.info("Starting upgrade of storage directory " + sd.getRoot());
115
116    // rename current to tmp
117    renameCurToTmp(sd);
118
119    final File curDir = sd.getCurrentDir();
120    final File tmpDir = sd.getPreviousTmp();
121    List<String> fileNameList = IOUtils.listDirectory(tmpDir, new FilenameFilter() {
122      @Override
123      public boolean accept(File dir, String name) {
124        return dir.equals(tmpDir)
125            && name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
126      }
127    });
128
129    for (String s : fileNameList) {
130      File prevFile = new File(tmpDir, s);
131      File newFile = new File(curDir, prevFile.getName());
132      Files.createLink(newFile.toPath(), prevFile.toPath());
133    }
134  }
135
136  /**
137   * Rename the existing current dir to previous.tmp, and create a new empty
138   * current dir.
139   */
140  public static void renameCurToTmp(StorageDirectory sd) throws IOException {
141    File curDir = sd.getCurrentDir();
142    File prevDir = sd.getPreviousDir();
143    final File tmpDir = sd.getPreviousTmp();
144
145    Preconditions.checkState(curDir.exists(),
146        "Current directory must exist for preupgrade.");
147    Preconditions.checkState(!prevDir.exists(),
148        "Previous directory must not exist for preupgrade.");
149    Preconditions.checkState(!tmpDir.exists(),
150        "Previous.tmp directory must not exist for preupgrade."
151            + "Consider restarting for recovery.");
152
153    // rename current to tmp
154    NNStorage.rename(curDir, tmpDir);
155
156    if (!curDir.mkdir()) {
157      throw new IOException("Cannot create directory " + curDir);
158    }
159  }
160  
161  /**
162   * Perform the upgrade of the storage dir to the given storage info. The new
163   * storage info is written into the current directory, and the previous.tmp
164   * directory is renamed to previous.
165   * 
166   * @param sd the storage directory to upgrade
167   * @param storage info about the new upgraded versions.
168   * @throws IOException in the event of error
169   */
170  public static void doUpgrade(StorageDirectory sd, Storage storage)
171      throws IOException {
172    LOG.info("Performing upgrade of storage directory " + sd.getRoot());
173    try {
174      // Write the version file, since saveFsImage only makes the
175      // fsimage_<txid>, and the directory is otherwise empty.
176      storage.writeProperties(sd);
177
178      File prevDir = sd.getPreviousDir();
179      File tmpDir = sd.getPreviousTmp();
180      Preconditions.checkState(!prevDir.exists(),
181          "previous directory must not exist for upgrade.");
182      Preconditions.checkState(tmpDir.exists(),
183          "previous.tmp directory must exist for upgrade.");
184
185      // rename tmp to previous
186      NNStorage.rename(tmpDir, prevDir);
187    } catch (IOException ioe) {
188      LOG.error("Unable to rename temp to previous for " + sd.getRoot(), ioe);
189      throw ioe;
190    }
191  }
192
193  /**
194   * Perform rollback of the storage dir to the previous state. The existing
195   * current dir is removed, and the previous dir is renamed to current.
196   * 
197   * @param sd the storage directory to roll back.
198   * @throws IOException in the event of error
199   */
200  static void doRollBack(StorageDirectory sd)
201      throws IOException {
202    File prevDir = sd.getPreviousDir();
203    if (!prevDir.exists()) {
204      return;
205    }
206
207    File tmpDir = sd.getRemovedTmp();
208    Preconditions.checkState(!tmpDir.exists(),
209        "removed.tmp directory must not exist for rollback."
210            + "Consider restarting for recovery.");
211    // rename current to tmp
212    File curDir = sd.getCurrentDir();
213    Preconditions.checkState(curDir.exists(),
214        "Current directory must exist for rollback.");
215
216    NNStorage.rename(curDir, tmpDir);
217    // rename previous to current
218    NNStorage.rename(prevDir, curDir);
219
220    // delete tmp dir
221    NNStorage.deleteDir(tmpDir);
222    LOG.info("Rollback of " + sd.getRoot() + " is complete.");
223  }
224  
225}