* Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * See the License for the specific language governing permissions and
 * limitations under the License.
package org.apache.hadoop.hdfs;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.Random;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.raid.RaidCodec;
import org.apache.hadoop.raid.RaidCodecBuilder;
import org.apache.hadoop.security.UnixUserGroupInformation;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class TestMergeFile {
  public static final Log LOG = LogFactory.getLog(TestMergeFile.class);

  private static final short REPL_FACTOR = 2;
  private MiniDFSCluster cluster;
  private NameNode nn;
  private DistributedFileSystem dfs;
  private DistributedFileSystem userdfs;
  private static long blockSize = 512;
  private static int numDataBlocks = 6;
  private static int numRSParityBlocks = 3;
  private static Configuration conf;
  private static Random rand = new Random();
  private static UnixUserGroupInformation USER1;
  private static int id = 0;
  static {
    conf = new Configuration();
    conf.setLong("dfs.block.size", blockSize);
    conf.setBoolean("dfs.permissions", true);
  public void startUpCluster() throws IOException {
    RaidCodecBuilder.loadDefaultFullBlocksCodecs(conf, numRSParityBlocks,
    cluster = new MiniDFSCluster(conf, REPL_FACTOR, true, null);
    assertNotNull("Failed Cluster Creation", cluster);
    dfs = (DistributedFileSystem) cluster.getFileSystem();
    assertNotNull("Failed to get FileSystem", dfs);
    nn = cluster.getNameNode();
    assertNotNull("Failed to get NameNode", nn);
    Configuration newConf = new Configuration(conf);
    USER1 = new UnixUserGroupInformation("foo", new String[] {"bar" });
        UnixUserGroupInformation.UGI_PROPERTY_NAME, USER1);
    userdfs = (DistributedFileSystem)FileSystem.get(newConf); // login as ugi

  public void shutDownCluster() throws IOException {
    if(dfs != null) {
    if (userdfs != null) {
    if(cluster != null) {
  public void mergeFile(Path parity, Path source, String codecId, int[] checksums,
      String exceptionMessage) throws Exception {
    mergeFile(dfs, parity, source, codecId, checksums, exceptionMessage);
  public void mergeFile(DistributedFileSystem fs, Path parity, Path source, 
      String codecId, int[] checksums, String exceptionMessage) throws Exception {
    try {
      fs.merge(parity, source, codecId, checksums);
    } catch (Exception e) {
      if (exceptionMessage == null) {
        // This is not expected
        throw e;
      assertTrue("Exception " + e.getMessage() + " doesn't match " + 
                 exceptionMessage, e.getMessage().contains(exceptionMessage));
  public void testMergeXORFile() throws Exception {
    mergeFile(12, 2, (short)2, "xor");
    mergeFile(9, 2, (short)1, "xor");
    mergeFile(3, 1, (short)2, "xor");
  public void testMergeRSFile() throws Exception {
    mergeFile(12, 6, (short)1, "rs");
    mergeFile(9, 6, (short)1, "rs");
    mergeFile(3, 3, (short)2, "rs");
   * @return current file status of file
  public static FileStatus verifyMergeFiles(DistributedFileSystem fileSys, FileStatus statBefore, 
      LocatedBlocks lbsBefore, Path source, long fileLen, long crc) throws Exception {
    FileStatus statAfter  = fileSys.getFileStatus(source);
    LocatedBlocks lbsAfter = fileSys.getLocatedBlocks(source, 0, fileLen);
    // Verify file stat
    assertEquals(statBefore.getBlockSize(), statAfter.getBlockSize());
    assertEquals(statBefore.getLen(), statAfter.getLen());
    assertEquals(statBefore.getReplication(), statAfter.getReplication());
    // Verify getLocatedBlocks
    for (int i = 0; i < lbsBefore.getLocatedBlocks().size(); i++) {
      assertEquals(lbsBefore.get(i).getBlock(), lbsAfter.get(i).getBlock());
    // Verify file content
    assertTrue("File content matches", DFSTestUtil.validateFile(fileSys, 
        statBefore.getPath(), statBefore.getLen(), crc));
    return statAfter;
  public void mergeFile(int numBlocks, int parityBlocks, short sourceRepl,
      String codecId) throws Exception {
    LOG.info("RUNNING testMergeFile numBlocks=" + numBlocks + 
        " parityBlocks=" + parityBlocks + " sourceRepl=" + sourceRepl +
        " codecId=" + codecId);
    long fileLen = blockSize * numBlocks;
    long parityLen = blockSize * parityBlocks;
    Path dir = new Path ("/user/facebook" + id);
    Path source = new Path(dir, "1");
    Path dest = new Path(dir, "2");
    long crc = DFSTestUtil.createFile(dfs, source, fileLen, sourceRepl, 1);
    Path parityDir = new Path("/raid/user/facebook" + id);
    RaidCodec codec = RaidCodec.getCodec(codecId);
    Path parity = new Path(parityDir, "1");
    DFSTestUtil.createFile(dfs, parity, parityLen,
        codec.parityReplication, 1);
    int[] checksums = new int[numBlocks];
    for (int i = 0; i < numBlocks; i++) {
      checksums[i] = rand.nextInt();
    ContentSummary cBefore = dfs.getContentSummary(dir);
    ContentSummary cParityBefore = dfs.getContentSummary(parityDir);
    FileStatus statBefore = dfs.getFileStatus(source);
    LocatedBlocks lbsBefore = dfs.getLocatedBlocks(source, 0, fileLen);
    dfs.setTimes(parity, statBefore.getModificationTime(), 0);
    // now merge
    dfs.merge(parity, source, codecId, checksums);
    ContentSummary cAfter = dfs.getContentSummary(dir);
    ContentSummary cParityAfter = dfs.getContentSummary(parityDir);
    // verify directory stat
    assertEquals("File count doesn't change", cBefore.getFileCount(),
    assertEquals("Space consumed is increased", 
        cBefore.getSpaceConsumed() + parityLen * codec.parityReplication,
    assertEquals("Parity file is removed", cParityBefore.getFileCount() - 1,
    assertEquals("Space consumed is 0", 0, cParityAfter.getSpaceConsumed());
    // Verify parity is removed
    verifyMergeFiles(dfs, statBefore, lbsBefore, source, fileLen, crc);
    LocatedBlocks lbsAfter = dfs.getLocatedBlocks(source, blockSize, fileLen);
    assertEquals(numBlocks - 1, lbsAfter.getLocatedBlocks().size());
    for (int i = 0; i < numBlocks - 1; i++) {
      assertEquals(lbsBefore.get(i + 1).getBlock(), lbsAfter.get(i).getBlock());
    assertTrue("Should not be able to hardlink a raided file", 
        !dfs.hardLink(source, dest));
  public void testMergeIllegalCases() throws Exception {
    LOG.info("Running testMergeIllegalCases");
    int numBlocks = 6;
    long fileLen = blockSize * numBlocks;
    Path dir = new Path ("/user/facebook");
    Path source = new Path(dir, "1");
    Path dest = new Path(dir, "2");
    DFSTestUtil.createFile(dfs, source, fileLen, REPL_FACTOR, 1);
    FileStatus stat = dfs.getFileStatus(source);
    Path raidDir = new Path("/raid/user/facebook");
    Path badParity = new Path(raidDir, "1");
    DFSTestUtil.createFile(dfs, badParity, blockSize * 2,
        (short)1, 1);
    int[] checksums = new int[numBlocks];
    for (int i = 0; i < numBlocks; i++) {
      checksums[i] = rand.nextInt();
    Path emptyFile = new Path("/empty");
    DFSTestUtil.createFile(dfs, emptyFile, 0L, REPL_FACTOR, 1);
    mergeFile(badParity, source, "xor", null, 
        "merge: checksum array is empty or null");
    mergeFile(badParity, source, "nonexist", checksums,
        "merge: codec nonexist doesn't exist");
    dfs.setOwner(source, "foo", "bar");
    dfs.setOwner(badParity, "foo", "bar");
    dfs.setOwner(raidDir, "foo", "bar");
    LOG.info("Disallow write on " + source);
    dfs.setPermission(source, new FsPermission((short)0577));
    mergeFile(userdfs, badParity, source, "rs", checksums, "Permission denied");
    LOG.info("Enable write on " + source + " and disable read on " + badParity);
    dfs.setPermission(source, new FsPermission((short)0777));
    dfs.setPermission(badParity, new FsPermission((short)0377));
    mergeFile(userdfs, badParity, source, "rs", checksums, "Permission denied");
    LOG.info("Enable read on " + badParity + " and disable write on " + raidDir);
    dfs.setPermission(badParity, new FsPermission((short)0777));
    dfs.setPermission(raidDir, new FsPermission((short)0577));
    mergeFile(userdfs, badParity, source, "rs", checksums, "Permission denied");
    dfs.setPermission(raidDir, new FsPermission((short)0777));
    LOG.info("Test different types of files");
    mergeFile(new Path("/nonexist"), source, "rs", checksums,
        "merge: source file or parity file doesn't exist");
    mergeFile(badParity, new Path("/nonexist"), "rs", checksums,
        "merge: source file or parity file doesn't exist");
    mergeFile(raidDir, source, "rs", checksums,
        "merge: source file or parity file is a directory");
    mergeFile(badParity, dir, "rs", checksums,
        "merge: source file or parity file is a directory");
    LOG.info("Set modification time of parity to be a different number");
    dfs.setTimes(badParity, stat.getModificationTime() + 1, 0);
    mergeFile(badParity, source, "rs", checksums,
        "merge: source file and parity file doesn't have the same modification time");
    dfs.setTimes(badParity, stat.getModificationTime(), 0);
    dfs.setTimes(emptyFile, stat.getModificationTime(), 0);
    mergeFile(emptyFile, source, "rs", checksums, 
        "merge: parity file's replication doesn't match codec's parity replication");
    dfs.setReplication(emptyFile, (short)1);
    mergeFile(emptyFile, source, "rs", checksums, "merge: /empty is empty");
    mergeFile(badParity, emptyFile, "rs", checksums, "merge: /empty is empty");
    mergeFile(badParity, source, "rs", new int[5], "merge: checksum length ");
    mergeFile(badParity, source, "rs", checksums, "merge: expect parity blocks ");
    LOG.info("Hardlink the file to " + dest);
    dfs.hardLink(source, dest);
    mergeFile(emptyFile, source, "rs", checksums, 
        "merge: source file or parity file is hardlinked");
    mergeFile(dest, emptyFile, "rs", checksums, 
        "merge: source file or parity file is hardlinked");