org.apache.hadoop.fs.AbstractFileSystem

Here are the examples of the java api class org.apache.hadoop.fs.AbstractFileSystem taken from open source projects.

1. HadoopSortingTest#testSortSimple()

Project: ignite
Source File: HadoopSortingTest.java
View license
/**
     * @throws Exception If failed.
     */
public void testSortSimple() throws Exception {
    // Generate test data.
    Job job = Job.getInstance();
    job.setInputFormatClass(InFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);
    job.setMapperClass(Mapper.class);
    job.setNumReduceTasks(0);
    setupFileSystems(job.getConfiguration());
    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_INPUT));
    X.printerrln("Data generation started.");
    grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration())).get(180000);
    X.printerrln("Data generation complete.");
    // Run main map-reduce job.
    job = Job.getInstance();
    setupFileSystems(job.getConfiguration());
    job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());
    FileInputFormat.setInputPaths(job, new Path(igfsScheme() + PATH_INPUT));
    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));
    job.setSortComparatorClass(JavaSerializationComparator.class);
    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);
    job.setNumReduceTasks(2);
    job.setMapOutputKeyClass(UUID.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);
    X.printerrln("Job started.");
    grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration())).get(180000);
    X.printerrln("Job complete.");
    // Check result.
    Path outDir = new Path(igfsScheme() + PATH_OUTPUT);
    AbstractFileSystem fs = AbstractFileSystem.get(new URI(igfsScheme()), job.getConfiguration());
    for (FileStatus file : fs.listStatus(outDir)) {
        X.printerrln("__ file: " + file);
        if (file.getLen() == 0)
            continue;
        FSDataInputStream in = fs.open(file.getPath());
        Scanner sc = new Scanner(in);
        UUID prev = null;
        while (sc.hasNextLine()) {
            UUID next = UUID.fromString(sc.nextLine());
            if (prev != null)
                assertTrue(prev.compareTo(next) < 0);
            prev = next;
        }
    }
}