DistCpV1.java 文件源码

/** Check whether the file list have duplication. */
static private void checkDuplication(FileSystem fs, Path file, Path sorted,
  Configuration conf) throws IOException {
  SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
    new Text.Comparator(), Text.class, Text.class, conf);
  sorter.sort(file, sorted);
  try (SequenceFile.Reader in =
       new SequenceFile.Reader(conf, Reader.file(sorted))) {
    Text prevdst = null, curdst = new Text();
    Text prevsrc = null, cursrc = new Text(); 
    for(; in.next(curdst, cursrc); ) {
      if (prevdst != null && curdst.equals(prevdst)) {
        throw new DuplicationException(
          "Invalid input, there are duplicated files in the sources: "
          + prevsrc + ", " + cursrc);
      }
      prevdst = curdst;
      curdst = new Text();
      prevsrc = cursrc;
      cursrc = new Text();
    }
  }
}