001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode;
019
020import java.io.File;
021import java.io.FilenameFilter;
022import java.io.IOException;
023import java.nio.file.Files;
024import java.util.List;
025
026import org.apache.commons.logging.Log;
027import org.apache.commons.logging.LogFactory;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.hdfs.server.common.Storage;
030import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
031import org.apache.hadoop.hdfs.server.common.StorageInfo;
032
033import com.google.common.base.Preconditions;
034
035public abstract class NNUpgradeUtil {
036  
037  private static final Log LOG = LogFactory.getLog(NNUpgradeUtil.class);
038  
039  /**
040   * Return true if this storage dir can roll back to the previous storage
041   * state, false otherwise. The NN will refuse to run the rollback operation
042   * unless at least one JM or fsimage storage directory can roll back.
043   * 
044   * @param storage the storage info for the current state
045   * @param prevStorage the storage info for the previous (unupgraded) state
046   * @param targetLayoutVersion the layout version we intend to roll back to
047   * @return true if this JM can roll back, false otherwise.
048   * @throws IOException in the event of error
049   */
050  static boolean canRollBack(StorageDirectory sd, StorageInfo storage,
051      StorageInfo prevStorage, int targetLayoutVersion) throws IOException {
052    File prevDir = sd.getPreviousDir();
053    if (!prevDir.exists()) {  // use current directory then
054      LOG.info("Storage directory " + sd.getRoot()
055               + " does not contain previous fs state.");
056      // read and verify consistency with other directories
057      storage.readProperties(sd);
058      return false;
059    }
060
061    // read and verify consistency of the prev dir
062    prevStorage.readPreviousVersionProperties(sd);
063
064    if (prevStorage.getLayoutVersion() != targetLayoutVersion) {
065      throw new IOException(
066        "Cannot rollback to storage version " +
067        prevStorage.getLayoutVersion() +
068        " using this version of the NameNode, which uses storage version " +
069        targetLayoutVersion + ". " +
070        "Please use the previous version of HDFS to perform the rollback.");
071    }
072    
073    return true;
074  }
075
076  /**
077   * Finalize the upgrade. The previous dir, if any, will be renamed and
078   * removed. After this is completed, rollback is no longer allowed.
079   * 
080   * @param sd the storage directory to finalize
081   * @throws IOException in the event of error
082   */
083  static void doFinalize(StorageDirectory sd) throws IOException {
084    File prevDir = sd.getPreviousDir();
085    if (!prevDir.exists()) { // already discarded
086      LOG.info("Directory " + prevDir + " does not exist.");
087      LOG.info("Finalize upgrade for " + sd.getRoot()+ " is not required.");
088      return;
089    }
090    LOG.info("Finalizing upgrade of storage directory " + sd.getRoot());
091    Preconditions.checkState(sd.getCurrentDir().exists(),
092        "Current directory must exist.");
093    final File tmpDir = sd.getFinalizedTmp();
094    // rename previous to tmp and remove
095    NNStorage.rename(prevDir, tmpDir);
096    NNStorage.deleteDir(tmpDir);
097    LOG.info("Finalize upgrade for " + sd.getRoot()+ " is complete.");
098  }
099  
100  /**
101   * Perform any steps that must succeed across all storage dirs/JournalManagers
102   * involved in an upgrade before proceeding onto the actual upgrade stage. If
103   * a call to any JM's or local storage dir's doPreUpgrade method fails, then
104   * doUpgrade will not be called for any JM. The existing current dir is
105   * renamed to previous.tmp, and then a new, empty current dir is created.
106   *
107   * @param conf configuration for creating {@link EditLogFileOutputStream}
108   * @param sd the storage directory to perform the pre-upgrade procedure.
109   * @throws IOException in the event of error
110   */
111  static void doPreUpgrade(Configuration conf, StorageDirectory sd)
112      throws IOException {
113    LOG.info("Starting upgrade of storage directory " + sd.getRoot());
114
115    // rename current to tmp
116    renameCurToTmp(sd);
117
118    final File curDir = sd.getCurrentDir();
119    final File tmpDir = sd.getPreviousTmp();
120    String[] fileNameList = tmpDir.list(new FilenameFilter() {
121      @Override
122      public boolean accept(File dir, String name) {
123        return dir.equals(tmpDir)
124            && name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
125      }
126    });
127    if (fileNameList == null) {
128      throw new IOException("either " + tmpDir +
129          " is not a directory or there is an I/O error");
130    }
131
132    for (String s : fileNameList) {
133      File prevFile = new File(tmpDir, s);
134      File newFile = new File(curDir, prevFile.getName());
135      Files.createLink(newFile.toPath(), prevFile.toPath());
136    }
137  }
138
139  /**
140   * Rename the existing current dir to previous.tmp, and create a new empty
141   * current dir.
142   */
143  public static void renameCurToTmp(StorageDirectory sd) throws IOException {
144    File curDir = sd.getCurrentDir();
145    File prevDir = sd.getPreviousDir();
146    final File tmpDir = sd.getPreviousTmp();
147
148    Preconditions.checkState(curDir.exists(),
149        "Current directory must exist for preupgrade.");
150    Preconditions.checkState(!prevDir.exists(),
151        "Previous directory must not exist for preupgrade.");
152    Preconditions.checkState(!tmpDir.exists(),
153        "Previous.tmp directory must not exist for preupgrade."
154            + "Consider restarting for recovery.");
155
156    // rename current to tmp
157    NNStorage.rename(curDir, tmpDir);
158
159    if (!curDir.mkdir()) {
160      throw new IOException("Cannot create directory " + curDir);
161    }
162  }
163  
164  /**
165   * Perform the upgrade of the storage dir to the given storage info. The new
166   * storage info is written into the current directory, and the previous.tmp
167   * directory is renamed to previous.
168   * 
169   * @param sd the storage directory to upgrade
170   * @param storage info about the new upgraded versions.
171   * @throws IOException in the event of error
172   */
173  public static void doUpgrade(StorageDirectory sd, Storage storage)
174      throws IOException {
175    LOG.info("Performing upgrade of storage directory " + sd.getRoot());
176    try {
177      // Write the version file, since saveFsImage only makes the
178      // fsimage_<txid>, and the directory is otherwise empty.
179      storage.writeProperties(sd);
180
181      File prevDir = sd.getPreviousDir();
182      File tmpDir = sd.getPreviousTmp();
183      Preconditions.checkState(!prevDir.exists(),
184          "previous directory must not exist for upgrade.");
185      Preconditions.checkState(tmpDir.exists(),
186          "previous.tmp directory must exist for upgrade.");
187
188      // rename tmp to previous
189      NNStorage.rename(tmpDir, prevDir);
190    } catch (IOException ioe) {
191      LOG.error("Unable to rename temp to previous for " + sd.getRoot(), ioe);
192      throw ioe;
193    }
194  }
195
196  /**
197   * Perform rollback of the storage dir to the previous state. The existing
198   * current dir is removed, and the previous dir is renamed to current.
199   * 
200   * @param sd the storage directory to roll back.
201   * @throws IOException in the event of error
202   */
203  static void doRollBack(StorageDirectory sd)
204      throws IOException {
205    File prevDir = sd.getPreviousDir();
206    if (!prevDir.exists()) {
207      return;
208    }
209
210    File tmpDir = sd.getRemovedTmp();
211    Preconditions.checkState(!tmpDir.exists(),
212        "removed.tmp directory must not exist for rollback."
213            + "Consider restarting for recovery.");
214    // rename current to tmp
215    File curDir = sd.getCurrentDir();
216    Preconditions.checkState(curDir.exists(),
217        "Current directory must exist for rollback.");
218
219    NNStorage.rename(curDir, tmpDir);
220    // rename previous to current
221    NNStorage.rename(prevDir, curDir);
222
223    // delete tmp dir
224    NNStorage.deleteDir(tmpDir);
225    LOG.info("Rollback of " + sd.getRoot() + " is complete.");
226  }
227  
228}