java类org.apache.hadoop.io.SequenceFile.Metadata的实例源码

@SuppressWarnings("deprecation")
private SequenceFile.Metadata readMetadata(FileSystem fs, Path file)
  throws IOException {
  LOG.info("reading file: " + file.toString());
  SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
  SequenceFile.Metadata meta = reader.getMetadata(); 
  reader.close();
  return meta;
}

private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata)
  throws IOException {
  fs.delete(sortedFile, true);
  LOG.info("sorting: " + unsortedFile + " to: " + sortedFile);
  final WritableComparator comparator = WritableComparator.get(RandomDatum.class);
  SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, RandomDatum.class, RandomDatum.class, conf, metadata);
  sorter.sort(new Path[] { unsortedFile }, sortedFile, false);
}

/**
 * Test that makes sure createWriter succeeds on a file that was 
 * already created
 * @throws IOException
 */
@SuppressWarnings("deprecation")
@Test
public void testCreateWriterOnExistingFile() throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.getLocal(conf);
  Path name = new Path(new Path(System.getProperty("test.build.data","."),
      "createWriterOnExistingFile") , "file");

  fs.create(name);
  SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
      RandomDatum.class, 512, (short) 1, 4096, false,
      CompressionType.NONE, null, new Metadata());
}

private SequenceFile.Metadata readMetadata(FileSystem fs, Path file)
  throws IOException {
  LOG.info("reading file: " + file.toString());
  SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
  SequenceFile.Metadata meta = reader.getMetadata(); 
  reader.close();
  return meta;
}

private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata)
  throws IOException {
  fs.delete(sortedFile, true);
  LOG.info("sorting: " + unsortedFile + " to: " + sortedFile);
  final WritableComparator comparator = WritableComparator.get(RandomDatum.class);
  SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, RandomDatum.class, RandomDatum.class, conf, metadata);
  sorter.sort(new Path[] { unsortedFile }, sortedFile, false);
}

/**
 * Test that makes sure createWriter succeeds on a file that was 
 * already created
 * @throws IOException
 */
public void testCreateWriterOnExistingFile() throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.getLocal(conf);
  Path name = new Path(new Path(System.getProperty("test.build.data","."),
      "createWriterOnExistingFile") , "file");

  fs.create(name);
  SequenceFile.createWriter(fs, conf, name, RandomDatum.class,
      RandomDatum.class, 512, (short) 1, 4096, false,
      CompressionType.NONE, null, new Metadata());
}

/**
 * Create sequence file Metadata for our WAL file with version and compression
 * type (if any).
 * @param conf
 * @param compress
 * @return Metadata instance.
 */
private static Metadata createMetadata(final Configuration conf,
    final boolean compress) {
  TreeMap<Text, Text> metaMap = new TreeMap<Text, Text>();
  metaMap.put(WAL_VERSION_KEY, WAL_VERSION);
  if (compress) {
    // Currently we only do one compression type.
    metaMap.put(WAL_COMPRESSION_TYPE_KEY, DICTIONARY_COMPRESSION_TYPE);
  }
  return new Metadata(metaMap);
}

/**
 * Call this method after init() has been executed
 * 
 * @return whether WAL compression is enabled
 */
static boolean isWALCompressionEnabled(final Metadata metadata) {
  // Check version is >= VERSION?
  Text txt = metadata.get(WAL_VERSION_KEY);
  if (txt == null || Integer.parseInt(txt.toString()) < COMPRESSION_VERSION) {
    return false;
  }
  // Now check that compression type is present.  Currently only one value.
  txt = metadata.get(WAL_COMPRESSION_TYPE_KEY);
  return txt != null && txt.equals(DICTIONARY_COMPRESSION_TYPE);
}

private SequenceFile.Metadata readMetadata(FileSystem fs, Path file)
  throws IOException {
  LOG.info("reading file: " + file.toString());
  SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
  SequenceFile.Metadata meta = reader.getMetadata(); 
  reader.close();
  return meta;
}

private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata)
  throws IOException {
  fs.delete(sortedFile, true);
  LOG.info("sorting: " + unsortedFile + " to: " + sortedFile);
  final WritableComparator comparator = WritableComparator.get(RandomDatum.class);
  SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, RandomDatum.class, RandomDatum.class, conf, metadata);
  sorter.sort(new Path[] { unsortedFile }, sortedFile, false);
}