org.apache.hadoop.fs.LocatedFileStatus

Here are the examples of the java api org.apache.hadoop.fs.LocatedFileStatus taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

60 Examples 7

19 Source : HistoryLogUtilsTest.java
with Apache License 2.0
from spotify

private LocatedFileStatus mockFileStatus(String filePath) {
    LocatedFileStatus mockLFS = Mockito.mock(LocatedFileStatus.clreplaced);
    Mockito.when(mockLFS.getPath()).thenReturn(new Path(filePath));
    return mockLFS;
}

19 Source : ViewFsLocatedFileStatus.java
with Apache License 2.0
from NJUJYB

clreplaced ViewFsLocatedFileStatus extends LocatedFileStatus {

    final LocatedFileStatus myFs;

    Path modifiedPath;

    ViewFsLocatedFileStatus(LocatedFileStatus locatedFileStatus, Path path) {
        myFs = locatedFileStatus;
        modifiedPath = path;
    }

    @Override
    public long getLen() {
        return myFs.getLen();
    }

    @Override
    public boolean isFile() {
        return myFs.isFile();
    }

    @Override
    public boolean isDirectory() {
        return myFs.isDirectory();
    }

    @Override
    @SuppressWarnings("deprecation")
    public boolean isDir() {
        return myFs.isDirectory();
    }

    @Override
    public boolean isSymlink() {
        return myFs.isSymlink();
    }

    @Override
    public long getBlockSize() {
        return myFs.getBlockSize();
    }

    @Override
    public short getReplication() {
        return myFs.getReplication();
    }

    @Override
    public long getModificationTime() {
        return myFs.getModificationTime();
    }

    @Override
    public long getAccessTime() {
        return myFs.getAccessTime();
    }

    @Override
    public FsPermission getPermission() {
        return myFs.getPermission();
    }

    @Override
    public String getOwner() {
        return myFs.getOwner();
    }

    @Override
    public String getGroup() {
        return myFs.getGroup();
    }

    @Override
    public Path getPath() {
        return modifiedPath;
    }

    @Override
    public void setPath(final Path p) {
        modifiedPath = p;
    }

    @Override
    public Path getSymlink() throws IOException {
        return myFs.getSymlink();
    }

    @Override
    public void setSymlink(Path p) {
        myFs.setSymlink(p);
    }

    @Override
    public BlockLocation[] getBlockLocations() {
        return myFs.getBlockLocations();
    }

    @Override
    public int compareTo(Object o) {
        return super.compareTo(o);
    }

    @Override
    public boolean equals(Object o) {
        return super.equals(o);
    }

    @Override
    public int hashCode() {
        return super.hashCode();
    }
}

19 Source : FSAgent.java
with Apache License 2.0
from lealone

public List<String> listFiles(String dir) throws IOException {
    List<String> files = new ArrayList<>();
    Path path = new Path(dir);
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    if (!fileStatus.isDirectory()) {
        throw new FileNotFoundException("Cannot read directory " + dir);
    }
    RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
    while (it.hasNext()) {
        LocatedFileStatus lfs = it.next();
        files.add(lfs.getPath().getName());
    }
    return files;
}

19 Source : FSAgent.java
with Apache License 2.0
from lealone

public List<LocatedFileStatus> listFilesInfo(String dir) throws IOException {
    List<LocatedFileStatus> files = new ArrayList<>();
    Path path = new Path(dir);
    FileStatus fileStatus = fileSystem.getFileStatus(path);
    if (!fileStatus.isDirectory()) {
        throw new FileNotFoundException("Cannot read directory " + dir);
    }
    RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
    while (it.hasNext()) {
        LocatedFileStatus lfs = it.next();
        files.add(lfs);
    }
    return files;
}

19 Source : HDFSEasy.java
with Apache License 2.0
from apache

public List<LocatedFileStatus> listFiles(Path path) throws IOException {
    RemoteIterator<LocatedFileStatus> i = dfs.listFiles(path, false);
    List<LocatedFileStatus> retList = new ArrayList<>();
    while (i.hasNext()) {
        LocatedFileStatus locatedFileStatus = i.next();
        retList.add(locatedFileStatus);
    }
    return retList;
}

18 Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng

public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, int bucketNumber, Optional<DeleteDeltaLocations> deleteDeltaLocations) {
    return createInternalHiveSplit(status, OptionalInt.of(bucketNumber), false, deleteDeltaLocations, Optional.empty());
}

18 Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng

public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, boolean splittable, Optional<DeleteDeltaLocations> deleteDeltaLocations, Optional<Long> startRowOffsetOfFile) {
    return createInternalHiveSplit(status, OptionalInt.empty(), splittable, deleteDeltaLocations, startRowOffsetOfFile);
}

18 Source : GenerateData.java
with Apache License 2.0
from NJUJYB

static DataStatistics publishPlainDataStatistics(Configuration conf, Path inputDir) throws IOException {
    FileSystem fs = inputDir.getFileSystem(conf);
    // obtain input data file statuses
    long dataSize = 0;
    long fileCount = 0;
    RemoteIterator<LocatedFileStatus> iter = fs.listFiles(inputDir, true);
    PathFilter filter = new Utils.OutputFileUtils.OutputFilesFilter();
    while (iter.hasNext()) {
        LocatedFileStatus lStatus = iter.next();
        if (filter.accept(lStatus.getPath())) {
            dataSize += lStatus.getLen();
            ++fileCount;
        }
    }
    // publish the plain data statistics
    LOG.info("Total size of input data : " + StringUtils.humanReadableInt(dataSize));
    LOG.info("Total number of input data files : " + fileCount);
    return new DataStatistics(dataSize, fileCount, false);
}

18 Source : HDFSTest.java
with Apache License 2.0
from junneyang

@Test
public void testList() throws FileNotFoundException, IOException {
    Path f = new Path("/");
    RemoteIterator<LocatedFileStatus> files = fs.listFiles(f, true);
    while (files.hasNext()) {
        LocatedFileStatus file = (LocatedFileStatus) files.next();
        LOGGER.info("====={}", file.getPath());
    }
}

18 Source : CamelSinkHDFSITCase.java
with Apache License 2.0
from apache

private void printFile(LocatedFileStatus f, String matchString) {
    try {
        String contents = hdfsEasy.readFile(f.getPath());
        LOG.debug("Retrieved file {} with contents: {}", f.getPath(), contents);
        boolean contains = contents.contains(matchString);
        replacedertTrue(contains, "Unexpected content for the remote file " + f.getPath().getName() + " content: [" + contents + "] should contain [" + matchString + "]");
    } catch (IOException e) {
        LOG.debug("Reading returned file {} failed: {}", f.getPath(), e.getMessage());
        fail("I/O error: " + e.getMessage());
    }
}

17 Source : HiveFileIterator.java
with Apache License 2.0
from trinodb

@Override
protected LocatedFileStatus computeNext() {
    while (true) {
        while (remoteIterator.hasNext()) {
            LocatedFileStatus status = getLocatedFileStatus(remoteIterator);
            // Ignore hidden files and directories. Hive ignores files starting with _ and . as well.
            String fileName = status.getPath().getName();
            if (fileName.startsWith("_") || fileName.startsWith(".")) {
                continue;
            }
            if (status.isDirectory()) {
                switch(nestedDirectoryPolicy) {
                    case IGNORED:
                        continue;
                    case RECURSE:
                        paths.add(status.getPath());
                        continue;
                    case FAIL:
                        throw new NestedDirectoryNotAllowedException(status.getPath());
                }
            }
            return status;
        }
        if (paths.isEmpty()) {
            return endOfData();
        }
        remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst());
    }
}

17 Source : Cloudup.java
with Apache License 2.0
from steveloughran

/**
 * List the source files and build the list.
 * @return list of uploads
 * @throws IOException failure to list
 */
private List<UploadEntry> createUploadList() throws IOException {
    List<UploadEntry> uploads = new ArrayList<>();
    RemoteIterator<LocatedFileStatus> ri = sourceFS.listFiles(sourcePath, true);
    while (ri.hasNext()) {
        LocatedFileStatus status = ri.next();
        UploadEntry entry = new UploadEntry(status);
        entry.setDest(getFinalPath(status.getPath()));
        uploads.add(entry);
    }
    LOG.info("List {}", ri);
    if (ri instanceof Closeable) {
        ((Closeable) ri).close();
    }
    return uploads;
}

17 Source : SegmentHelper.java
with Apache License 2.0
from shunfei

public static void literalAllSegments(FileSystem fileSystem, Path dir, Consumer<LocatedFileStatus> consumer) throws IOException {
    RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(dir, true);
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        if (!fileStatus.isFile()) {
            continue;
        }
        if (fileStatus.getLen() == 0) {
            continue;
        }
        Path path = fileStatus.getPath();
        if (checkSegmentByPath(path)) {
            consumer.accept(fileStatus);
        }
    }
}

17 Source : HiveFileIterator.java
with Apache License 2.0
from openlookeng

@Override
protected LocatedFileStatus computeNext() {
    while (true) {
        while (remoteIterator.hasNext()) {
            LocatedFileStatus status = getLocatedFileStatus(remoteIterator);
            // Ignore hidden files and directories. Hive ignores files starting with _ and . as well.
            String fileName = status.getPath().getName();
            if (fileName.startsWith("_") || fileName.startsWith(".")) {
                continue;
            }
            if (status.isDirectory()) {
                switch(nestedDirectoryPolicy) {
                    case IGNORED:
                        continue;
                    case RECURSE:
                        paths.add(status.getPath());
                        continue;
                    case FAIL:
                        throw new NestedDirectoryNotAllowedException();
                }
            }
            return status;
        }
        if (paths.isEmpty()) {
            return endOfData();
        }
        remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst());
    }
}

17 Source : TestV2LsOperations.java
with Apache License 2.0
from NJUJYB

/**
 * To get this project to compile under Hadoop 1, this code needs to be
 * commented out
 *
 * @param fs filesystem
 * @param dir dir
 * @param subdir subdir
 * @param recursive recurse?
 * @throws IOException IO problems
 */
public static void replacedertListFilesFinds(FileSystem fs, Path dir, Path subdir, boolean recursive) throws IOException {
    RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(dir, recursive);
    boolean found = false;
    int entries = 0;
    StringBuilder builder = new StringBuilder();
    while (iterator.hasNext()) {
        LocatedFileStatus next = iterator.next();
        entries++;
        builder.append(next.toString()).append('\n');
        if (next.getPath().equals(subdir)) {
            found = true;
        }
    }
    replacedertTrue("Path " + subdir + " not found in directory " + dir + " : " + " entries=" + entries + " content" + builder.toString(), found);
}

17 Source : FilePaths.java
with Apache License 2.0
from ExpediaInceCommercePlatform

/**
 * Adds a file to the object
 *
 * @param file {@link LocatedFileStatus}
 */
public void addFile(final LocatedFileStatus file) {
    bytes += file.getLen();
    pathsByBytes.put(file.getPath(), file.getLen());
}

16 Source : FileSegmentPool.java
with Apache License 2.0
from shunfei

public void refreshLocalities() {
    try {
        // HashMap taks muti-thread risk here. Change to ConcurrentHashMap if it happens.
        Map<String, List<String>> newHostMap = new HashMap<>(segmentFdMap.size());
        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(segmentRootPath, true);
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            if (fileStatus.getLen() == 0) {
                continue;
            }
            String name = getSegmentName(fileStatus);
            if (name == null) {
                continue;
            }
            BlockLocation[] locations = fileStatus.getBlockLocations();
            if (locations.length != 1) {
                logger.error("A segment should only consisted by one block, now {}. Ignored: {}", locations.length, name);
                continue;
            }
            List<String> hosts = Arrays.asList(locations[0].getHosts());
            newHostMap.put(name, hosts);
        }
        hostMap = newHostMap;
    } catch (IOException e) {
        if (e instanceof ClosedByInterruptException) {
            logger.warn("Refresh [{}] segment locality failed by ClosedByInterruptException.", tableName);
            // Normally close interrupt.
            return;
        }
        String msg = e.getMessage();
        if (msg != null && Strings.equals(msg.trim(), "Filesystem closed")) {
            logger.warn("Refresh [{}] segment locality failed by Filesystem closed.", tableName);
            // Normally close interrupt.
            return;
        }
        logger.warn("Refresh [{}] segment locality failed.", tableName, e);
    }
}

16 Source : AbstractFlagConfig.java
with Apache License 2.0
from NationalSecurityAgency

protected Path getTestFile(FileSystem fs) throws IOException {
    createTestFiles(1, 1);
    Path file = null;
    for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(this.fmc.getBaseHDFSDir()), true); it.hasNext(); ) {
        LocatedFileStatus status = it.next();
        if (status.isFile()) {
            file = status.getPath();
            break;
        }
    }
    return file;
}

16 Source : RecordingsAgent.java
with Apache License 2.0
from lealone

public List<RecordingInfo> getRecordingInfo(String appId) {
    List<RecordingInfo> result = new ArrayList<>();
    String dir = getRecordingsDirectory(appId);
    if (dir == null) {
        return result;
    }
    Path path = new Path(dir);
    try {
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        if (!fileStatus.isDirectory()) {
            return result;
        }
        RemoteIterator<LocatedFileStatus> ri = stramAgent.getFileSystem().listLocatedStatus(path);
        while (ri.hasNext()) {
            LocatedFileStatus lfs = ri.next();
            if (lfs.isDirectory()) {
                try {
                    String opId = lfs.getPath().getName();
                    result.addAll(getRecordingInfo(appId, opId));
                } catch (NumberFormatException ex) {
                // ignore
                }
            }
        }
    } catch (IOException ex) {
        LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
        return result;
    }
    return result;
}

16 Source : RecordingsAgent.java
with Apache License 2.0
from lealone

private List<RecordingInfo> getRecordingInfoHelper(String appId, String opId, Set<String> containers) {
    List<RecordingInfo> result = new ArrayList<>();
    String dir = getRecordingsDirectory(appId, opId);
    if (dir == null) {
        return result;
    }
    Path path = new Path(dir);
    try {
        FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
        if (!fileStatus.isDirectory()) {
            return result;
        }
        RemoteIterator<LocatedFileStatus> ri = stramAgent.getFileSystem().listLocatedStatus(path);
        while (ri.hasNext()) {
            LocatedFileStatus lfs = ri.next();
            if (lfs.isDirectory()) {
                try {
                    String id = lfs.getPath().getName();
                    RecordingInfo recordingInfo = getRecordingInfoHelper(appId, opId, id, containers);
                    if (recordingInfo != null) {
                        result.add(recordingInfo);
                    }
                } catch (NumberFormatException ex) {
                // ignore
                }
            }
        }
    } catch (IOException ex) {
        LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
        return result;
    }
    return result;
}

16 Source : HadoopFileSystemUtils.java
with GNU General Public License v3.0
from icgc-dcc

private static List<LocatedFileStatus> getFiles(FileSystem fileSystem, Path target, boolean recusre) {
    val results = Lists.<LocatedFileStatus>newArrayList();
    RemoteIterator<LocatedFileStatus> fileStatusLisreplacederator = null;
    try {
        fileStatusLisreplacederator = fileSystem.listFiles(target, true);
        while (fileStatusLisreplacederator.hasNext()) {
            LocatedFileStatus fileStatus = fileStatusLisreplacederator.next();
            results.add(fileStatus);
        }
    } catch (IOException e) {
        log.info("Error retriving files in path '{}'", target);
    }
    return results;
}

16 Source : TestIcebergStreamWriter.java
with Apache License 2.0
from apache

private Set<String> scanDataFiles() throws IOException {
    Path dataDir = new Path(tablePath, "data");
    FileSystem fs = FileSystem.get(new Configuration());
    if (!fs.exists(dataDir)) {
        return ImmutableSet.of();
    } else {
        Set<String> paths = Sets.newHashSet();
        RemoteIterator<LocatedFileStatus> iterators = fs.listFiles(dataDir, true);
        while (iterators.hasNext()) {
            LocatedFileStatus status = iterators.next();
            if (status.isFile()) {
                Path path = status.getPath();
                if (path.getName().endsWith("." + format.toString().toLowerCase())) {
                    paths.add(path.toString());
                }
            }
        }
        return paths;
    }
}

15 Source : HdfsIOBenchmark.java
with Apache License 2.0
from zrlio

void browseDir() throws Exception {
    System.out.println("reading enumarate dir, path " + path);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    // benchmark
    System.out.println("starting benchmark...");
    RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, false);
    while (iter.hasNext()) {
        LocatedFileStatus status = iter.next();
        System.out.println(status.getPath());
    }
    fs.close();
}

15 Source : InternalHiveSplitFactory.java
with Apache License 2.0
from trinodb

public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable, Optional<AcidInfo> acidInfo) {
    splittable = splittable && status.getLen() > minimumTargetSplitSizeInBytes && isSplittable(inputFormat, fileSystem, status.getPath());
    return createInternalHiveSplit(status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), status.getModificationTime(), bucketNumber, splittable, acidInfo);
}

15 Source : ITestLocalCloudup.java
with Apache License 2.0
from steveloughran

@Test
public void testCopyRecursive() throws Throwable {
    int expected = createTestFiles(sourceDir, 64);
    expectSuccess(new Cloudup(), "-s", sourceDir.toURI().toString(), "-d", destDir.toURI().toString(), "-t", "4", "-l", "3");
    LocalFileSystem local = FileSystem.getLocal(new Configuration());
    Set<String> entries = new TreeSet<>();
    RemoteIterator<LocatedFileStatus> iterator = local.listFiles(new Path(destDir.toURI()), true);
    int count = 0;
    while (iterator.hasNext()) {
        LocatedFileStatus next = iterator.next();
        entries.add(next.getPath().toUri().toString());
        LOG.info("Entry {} size = {}", next.getPath(), next.getLen());
        count++;
    }
    replacedertEquals("Mismatch in files found", expected, count);
}

15 Source : ApplicationDriver.java
with GNU Lesser General Public License v3.0
from schic

private List<String> buildJarFiles(final MutableRef<String> primaryJarRef) throws IOException {
    final List<String> list = new ArrayList<>();
    final Path directoryPath = new Path(_jarDirectoryPath);
    final RemoteIterator<LocatedFileStatus> files = _fileSystem.listFiles(directoryPath, false);
    while (files.hasNext()) {
        final LocatedFileStatus file = files.next();
        final Path path = file.getPath();
        final String filename = path.getName();
        boolean primaryJar = false;
        for (final String prefix : PRIMARY_JAR_FILENAME_PREFIXES) {
            if (filename.startsWith(prefix)) {
                primaryJarRef.set(path.toString());
                primaryJar = true;
                break;
            }
        }
        if (!primaryJar) {
            list.add(path.toString());
        }
    }
    if (primaryJarRef.get() == null) {
        throw new IllegalArgumentException("Failed to find primary jar (starting with '" + PRIMARY_JAR_FILENAME_PREFIXES[0] + "') in JAR file directory: " + _jarDirectoryPath);
    }
    return list;
}

15 Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng

private Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable, Optional<DeleteDeltaLocations> deleteDeltaLocations, Optional<Long> startRowOffsetOfFile) {
    splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath());
    return createInternalHiveSplit(status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), status.getModificationTime(), bucketNumber, splittable, deleteDeltaLocations, startRowOffsetOfFile);
}

15 Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB

/**
 * Test when input patch has a symbolic links as its children
 */
@Test
public void testSymbolicLinks() throws IOException {
    writeFile(fc, FILE1, FILE_LEN);
    writeFile(fc, FILE2, FILE_LEN);
    writeFile(fc, FILE3, FILE_LEN);
    Path dir4 = new Path(TEST_DIR, "dir4");
    Path dir5 = new Path(dir4, "dir5");
    Path file4 = new Path(dir4, "file4");
    fc.createSymlink(DIR1, dir5, true);
    fc.createSymlink(FILE1, file4, true);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(dir4, true);
    LocatedFileStatus stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE3), stat.getPath());
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertFalse(itor.hasNext());
    itor = fc.util().listFiles(dir4, false);
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertFalse(itor.hasNext());
}

15 Source : QueryInputFormat.java
with Apache License 2.0
from Merck

public static void addQueryRecursively(Configuration conf, Path path, boolean sparqlUpdate, int stage) throws IOException {
    RemoteIterator<LocatedFileStatus> iter = path.getFileSystem(conf).listLocatedStatus(path);
    while (iter.hasNext()) {
        LocatedFileStatus stat = iter.next();
        if (stat.isDirectory()) {
            addQueryRecursively(conf, stat.getPath(), sparqlUpdate, stage);
        } else {
            addQuery(conf, stat, sparqlUpdate, stage);
        }
    }
}

15 Source : ColumnarFilesReader.java
with Apache License 2.0
from Kyligence

void checkPath() {
    try {
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(folderPath, false);
        if (files == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            Path path = fileStatus.getPath();
            String name = path.getName();
            if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
                dataFilePath = path;
            } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
                metaFilePath = path;
            } else {
                logger.warn("Contains invalid file {} in path {}", path, folderPath);
            }
        }
        if (dataFilePath == null || metaFilePath == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
    } catch (IOException e) {
        throw new RuntimeException("io error", e);
    }
}

15 Source : HDFSResourceStore.java
with Apache License 2.0
from Kyligence

@Override
protected void visitFolderImpl(String folderPath, boolean recursive, VisitFilter filter, boolean loadContent, Visitor visitor) throws IOException {
    Path p = getRealHDFSPath(folderPath);
    if (!fs.exists(p) || !fs.isDirectory(p)) {
        return;
    }
    String fsPathPrefix = p.toUri().getPath();
    String resPathPrefix = folderPath.endsWith("/") ? folderPath : folderPath + "/";
    RemoteIterator<LocatedFileStatus> it = fs.listFiles(p, recursive);
    while (it.hasNext()) {
        LocatedFileStatus status = it.next();
        if (status.isDirectory())
            continue;
        String path = status.getPath().toUri().getPath();
        if (!path.startsWith(fsPathPrefix))
            throw new IllegalStateException("File path " + path + " is supposed to start with " + fsPathPrefix);
        String resPath = resPathPrefix + path.substring(fsPathPrefix.length() + 1);
        if (filter.matches(resPath, status.getModificationTime())) {
            RawResource raw;
            if (loadContent)
                raw = new RawResource(resPath, status.getModificationTime(), fs.open(status.getPath()));
            else
                raw = new RawResource(resPath, status.getModificationTime());
            try {
                visitor.visit(raw);
            } finally {
                raw.close();
            }
        }
    }
}

15 Source : HadoopFileSystem.java
with Apache License 2.0
from DSC-SPIDAL

/**
 * List the statuses of the files/directories in the given path if the path is
 * a directory.
 *
 * @param f given path
 * @return the statuses of the files/directories in the given patch
 */
@Override
public FileStatus[] listFiles(Path f) throws IOException {
    RemoteIterator<LocatedFileStatus> listFiles = this.hadoopFileSystem.listFiles(toHadoopPath(f), true);
    List<FileStatus> statusList = new ArrayList<>();
    while (listFiles.hasNext()) {
        LocatedFileStatus next = listFiles.next();
        FileStatus status = new HadoopFileStatus(next);
        statusList.add(status);
    }
    return statusList.toArray(new FileStatus[0]);
}

15 Source : TestRemoteNodeFileSystemDual.java
with Apache License 2.0
from dremio

@Test
public void testClientWriteEmptyFile() throws Exception {
    Path basePath = new Path(temporaryFolder.newFolder().getAbsolutePath());
    Path path = ((PathCanonicalizer) clientFS).canonicalizePath(new Path(basePath, "testfile.bytes"));
    // create a file
    FSDataOutputStream stream = clientFS.create(path, false);
    // close it without writing anything to it
    stream.close();
    // make sure the file was created
    RemoteIterator<LocatedFileStatus> iter = client.fileSystem.listFiles(basePath, false);
    replacedertEquals(true, iter.hasNext());
    LocatedFileStatus status = iter.next();
    try (FSDataInputStream in = clientFS.open(status.getPath())) {
        in.readByte();
        fail("Fail is expected to be empty");
    } catch (EOFException e) {
    // empty file as expected
    }
    client.fileSystem.delete(status.getPath(), false);
}

14 Source : FileInputFormat.java
with Apache License 2.0
from NJUJYB

/**
 * Add files in the input path recursively into the results.
 * @param result
 *          The List to store all files.
 * @param fs
 *          The FileSystem.
 * @param path
 *          The input path.
 * @param inputFilter
 *          The input filter that can be used to filter files/dirs.
 * @throws IOException
 */
protected void addInputPathRecursively(List<FileStatus> result, FileSystem fs, Path path, PathFilter inputFilter) throws IOException {
    RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
    while (iter.hasNext()) {
        LocatedFileStatus stat = iter.next();
        if (inputFilter.accept(stat.getPath())) {
            if (stat.isDirectory()) {
                addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
            } else {
                result.add(stat);
            }
        }
    }
}

14 Source : BucketingSinkTest.java
with Apache License 2.0
from ljygz

/**
 * This uses {@link DateTimeBucketer} to
 * produce rolling files. We use {@link OneInputStreamOperatorTestHarness} to manually
 * advance processing time.
 */
@Test
public void testDateTimeRollingStringWriter() throws Exception {
    final int numElements = 20;
    final String outPath = hdfsURI + "/rolling-out";
    BucketingSink<String> sink = new BucketingSink<String>(outPath).setBucketer(new DateTimeBucketer<String>("ss")).setPartPrefix(PART_PREFIX).setPendingPrefix("").setPendingSuffix("");
    OneInputStreamOperatorTestHarness<String, Object> testHarness = createTestSink(sink, 1, 0);
    testHarness.setProcessingTime(0L);
    testHarness.setup();
    testHarness.open();
    for (int i = 0; i < numElements; i++) {
        // Every 5 elements, increase the clock time. We should end up with 5 elements per bucket.
        if (i % 5 == 0) {
            testHarness.setProcessingTime(i * 1000L);
        }
        testHarness.processElement(new StreamRecord<>("message #" + Integer.toString(i)));
    }
    testHarness.close();
    RemoteIterator<LocatedFileStatus> files = dfs.listFiles(new Path(outPath), true);
    // We should have 4 rolling files across 4 time intervals
    int numFiles = 0;
    while (files.hasNext()) {
        LocatedFileStatus file = files.next();
        numFiles++;
        if (file.getPath().toString().contains("rolling-out/00")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 0; i < 5; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/05")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 5; i < 10; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/10")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 10; i < 15; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/15")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 15; i < 20; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else {
            replacedert.fail("File " + file + " does not match any expected roll pattern.");
        }
    }
    replacedert.replacedertEquals(4, numFiles);
}

14 Source : ColumnarSplitInputFormat.java
with Apache License 2.0
from Kyligence

private boolean isValidFragmentPath(FileSystem fs, Path path) throws IOException {
    RemoteIterator<LocatedFileStatus> files = fs.listFiles(path, false);
    if (files == null) {
        logger.warn("Invalid fragment path:{}, empty folder", path);
        return false;
    }
    boolean hasDataFile = false;
    boolean hasMetaFile = false;
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        Path childPath = fileStatus.getPath();
        String name = childPath.getName();
        if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
            hasDataFile = true;
        } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
            hasMetaFile = true;
        } else {
            logger.warn("Contains invalid file {} in path {}", childPath, path);
        }
    }
    if (hasDataFile && hasMetaFile) {
        return true;
    } else {
        logger.warn("Invalid fragment path:{}, data file exists:{}, meta file exists:{}", path, hasDataFile, hasMetaFile);
        return false;
    }
}

14 Source : TestRemoteNodeFileSystemDual.java
with Apache License 2.0
from dremio

@Test
public void basicClientReadWrite() throws Exception {
    Path basePath = new Path(temporaryFolder.newFolder().getAbsolutePath());
    Path path = ((PathCanonicalizer) clientFS).canonicalizePath(new Path(basePath, "testfile.bytes"));
    final byte[] randomBytesMoreThanBuffer = new byte[RemoteNodeFileSystem.REMOTE_WRITE_BUFFER_SIZE_DEFAULT * 3];
    Random r = new Random();
    r.nextBytes(randomBytesMoreThanBuffer);
    try (FSDataOutputStream stream = clientFS.create(path, false)) {
        stream.write(randomBytesMoreThanBuffer);
    }
    RemoteIterator<LocatedFileStatus> iter = client.fileSystem.listFiles(basePath, false);
    replacedertEquals(true, iter.hasNext());
    LocatedFileStatus status = iter.next();
    try (FSDataInputStream in = clientFS.open(status.getPath())) {
        byte[] back = new byte[randomBytesMoreThanBuffer.length];
        int dataRead = in.read(back);
        replacedertEquals(back.length, dataRead);
        replacedertTrue(Arrays.equals(randomBytesMoreThanBuffer, back));
    }
    client.fileSystem.delete(status.getPath(), false);
}

14 Source : ProtoParquetWriterWithOffsetTest.java
with Apache License 2.0
from criteo

private List<EventHeaderProtos.Header> checkSingleFileWithFileSystem(Collection<EventHeaderProtos.Header> inputHeaders) throws IOException {
    final List<EventHeaderProtos.Header> headers = new LinkedList<>();
    Path newTmpFile = new Path(tmpPath, "file");
    final ProtoParquetWriter<Message> writer = new ProtoParquetWriter<>(newTmpFile, EventHeaderProtos.Header.clreplaced);
    long offset = 1;
    final BiConsumer<String, String> protoMetadataWriter = mock(BiConsumer.clreplaced);
    final ProtoParquetWriterWithOffset consumer = new ProtoParquetWriterWithOffset<>(writer, newTmpFile, finalPath, localFs, new FixedOffsetComputer(FINAL_FILE_NAME, 123), UTC_EPOCH, "ignored", protoMetadataWriter, 1);
    for (EventHeaderProtos.Header header : inputHeaders) {
        consumer.write(1234567890L, header, new TopicParreplacedionOffset(TOPIC, 1, offset++));
    }
    consumer.close();
    final RemoteIterator<LocatedFileStatus> filesIterator = localFs.listFiles(finalPath, false);
    final LocatedFileStatus fileStatus = filesIterator.next();
    replacedert.replacedertEquals(FINAL_FILE_NAME, fileStatus.getPath().getName());
    replacedert.replacedertFalse("There should be only one output file", filesIterator.hasNext());
    final ParquetReader<EventHeaderProtos.Header.Builder> reader;
    reader = ProtoParquetReader.<EventHeaderProtos.Header.Builder>builder(fileStatus.getPath()).build();
    EventHeaderProtos.Header.Builder current = reader.read();
    while (current != null) {
        headers.add(current.build());
        current = reader.read();
    }
    return headers;
}

14 Source : TestOzoneFileInterfaces.java
with Apache License 2.0
from apache

@Test
public void testOzoneManagerLocatedFileStatus() throws IOException {
    String data = RandomStringUtils.randomAlphanumeric(20);
    String filePath = RandomStringUtils.randomAlphanumeric(5);
    Path path = createPath("/" + filePath);
    try (FSDataOutputStream stream = fs.create(path)) {
        stream.writeBytes(data);
    }
    FileStatus status = fs.getFileStatus(path);
    replacedertTrue(status instanceof LocatedFileStatus);
    LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
    replacedertTrue(locatedFileStatus.getBlockLocations().length >= 1);
    for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
        replacedertTrue(blockLocation.getNames().length >= 1);
        replacedertTrue(blockLocation.getHosts().length >= 1);
    }
}

14 Source : TestOzoneFileInterfaces.java
with Apache License 2.0
from apache

@Test
@Ignore("HDDS-3506")
public void testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile() throws Exception {
    // naive replacedumption: MiniOzoneCluster will not have larger than ~1GB
    // block size when running this test.
    int blockSize = (int) fs.getConf().getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
    String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
    String filePath = RandomStringUtils.randomAlphanumeric(5);
    Path path = createPath("/" + filePath);
    try (FSDataOutputStream stream = fs.create(path)) {
        stream.writeBytes(data);
    }
    FileStatus status = fs.getFileStatus(path);
    replacedertTrue(status instanceof LocatedFileStatus);
    LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
    BlockLocation[] blockLocations = locatedFileStatus.getBlockLocations();
    replacedertEquals(0, blockLocations[0].getOffset());
    replacedertEquals(blockSize, blockLocations[1].getOffset());
    replacedertEquals(2 * blockSize, blockLocations[2].getOffset());
    replacedertEquals(blockSize, blockLocations[0].getLength());
    replacedertEquals(blockSize, blockLocations[1].getLength());
    replacedertEquals(837, blockLocations[2].getLength());
}

13 Source : TestCTTAS.java
with Apache License 2.0
from zpochen

private List<Path> findTemporaryTableLocation(String tableName) throws IOException {
    Path sessionTempLocation = new Path(getDfsTestTmpSchemaLocation(), session_id.toString());
    replacedertTrue("Session temporary location must exist", fs.exists(sessionTempLocation));
    replacedertEquals("Session temporary location permission should match", expectedFolderPermission, fs.getFileStatus(sessionTempLocation).getPermission());
    String tableUUID = UUID.nameUUIDFromBytes(tableName.getBytes()).toString();
    RemoteIterator<LocatedFileStatus> pathList = fs.listLocatedStatus(sessionTempLocation);
    List<Path> matchingPath = Lists.newArrayList();
    while (pathList.hasNext()) {
        LocatedFileStatus path = pathList.next();
        if (path.isDirectory() && path.getPath().getName().equals(tableUUID)) {
            matchingPath.add(path.getPath());
        }
    }
    return matchingPath;
}

13 Source : FileSegmentPool.java
with Apache License 2.0
from shunfei

private synchronized boolean doRefreshSegments() throws IOException {
    RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(segmentRootPath, true);
    Set<String> nameSet = new HashSet<>(2048);
    int updateCount = 0;
    boolean hasError = false;
    // Load segment from storage.
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        String name = getSegmentName(fileStatus);
        if (fileStatus.getLen() == 0 || name == null) {
            continue;
        }
        nameSet.add(name);
        long modifyTime = fileStatus.getModificationTime();
        SegmentFdAndTime st = segmentFdMap.get(name);
        if (st != null && st.modifyTime == modifyTime && st.fileSize == fileStatus.getLen()) {
            // Segment exist and is up to dated.
            continue;
        }
        BlockLocation[] locations = fileStatus.getBlockLocations();
        if (locations.length != 1) {
            logger.error("A segment should only consisted by one block, now {}. Ignored: {}", locations.length, name);
            continue;
        }
        Path segmentPath = segmentPath(name);
        ByteBufferReader.Opener readerOpener = ByteBufferReader.Opener.create(fileSystem, segmentPath, fileStatus.getLen(), locations.length);
        SegmentMeta sectionInfo = null;
        try (ByteBufferReader reader = readerOpener.open(0)) {
            sectionInfo = Integrate.INSTANCE.read(reader);
            if (sectionInfo == null) {
                // Not a segment.
                continue;
            }
            IntegratedSegment.Fd fd = IntegratedSegment.Fd.create(name, sectionInfo, readerOpener);
            if (fd.info().rowCount() == 0) {
                logger.debug("table [{}] ignore empty segment [{}]", tableName, name);
            } else {
                segmentFdMap.put(name, new SegmentFdAndTime(fd, fileStatus.getModificationTime(), fileStatus.getLen()));
                logger.info("table [{}] add new segment [{}]", tableName, name);
                updateCount++;
            }
        } catch (IOException e) {
            hasError = true;
            logger.error("", e);
            logger.error("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
            logger.error("Load segment [{}: {}] failed, system in inconsistent state", tableName, name);
            logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
        }
    }
    // Remove outdated segments.
    Iterator<String> it = segmentFdMap.keySet().iterator();
    while (it.hasNext()) {
        String name = it.next();
        if (!nameSet.contains(name)) {
            logger.info("table [{}] remove segment [{}]", tableName, name);
            it.remove();
            updateCount++;
        }
    }
    if (updateCount > 0) {
        updateSegmentFdList();
    }
    // Save to local cache.
    totalUpdateCount += updateCount;
    if (totalUpdateCount >= SaveCacheUpdateCount || lastSaveCacheTime + SaveCachePeriod <= System.currentTimeMillis() || mustRefresh) {
        Try.on(this::saveToLocalCache, 1, logger, String.format("Save %s segment fds to local cache failed", tableName));
        logger.debug("save [{}] segment fds to local cache", tableName);
        totalUpdateCount = 0;
        lastSaveCacheTime = System.currentTimeMillis();
    }
    mustRefresh = false;
    return !hasError;
}

13 Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB

/**
 * Test when input path is a directory
 */
@Test
public void testDirectory() throws IOException {
    fc.mkdir(DIR1, FsPermission.getDefault(), true);
    // test empty directory
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(DIR1, true);
    replacedertFalse(itor.hasNext());
    itor = fc.util().listFiles(DIR1, false);
    replacedertFalse(itor.hasNext());
    // testing directory with 1 file
    writeFile(fc, FILE2, FILE_LEN);
    itor = fc.util().listFiles(DIR1, true);
    LocatedFileStatus stat = itor.next();
    replacedertFalse(itor.hasNext());
    replacedertTrue(stat.isFile());
    replacedertEquals(FILE_LEN, stat.getLen());
    replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
    replacedertEquals(1, stat.getBlockLocations().length);
    itor = fc.util().listFiles(DIR1, false);
    stat = itor.next();
    replacedertFalse(itor.hasNext());
    replacedertTrue(stat.isFile());
    replacedertEquals(FILE_LEN, stat.getLen());
    replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
    replacedertEquals(1, stat.getBlockLocations().length);
    // test more complicated directory
    writeFile(fc, FILE1, FILE_LEN);
    writeFile(fc, FILE3, FILE_LEN);
    itor = fc.util().listFiles(TEST_DIR, true);
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE3), stat.getPath());
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertFalse(itor.hasNext());
    itor = fc.util().listFiles(TEST_DIR, false);
    stat = itor.next();
    replacedertTrue(stat.isFile());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertFalse(itor.hasNext());
}

13 Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB

/**
 * Test when input path is a file
 */
@Test
public void testFile() throws IOException {
    fc.mkdir(TEST_DIR, FsPermission.getDefault(), true);
    writeFile(fc, FILE1, FILE_LEN);
    RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(FILE1, true);
    LocatedFileStatus stat = itor.next();
    replacedertFalse(itor.hasNext());
    replacedertTrue(stat.isFile());
    replacedertEquals(FILE_LEN, stat.getLen());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertEquals(1, stat.getBlockLocations().length);
    itor = fc.util().listFiles(FILE1, false);
    stat = itor.next();
    replacedertFalse(itor.hasNext());
    replacedertTrue(stat.isFile());
    replacedertEquals(FILE_LEN, stat.getLen());
    replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
    replacedertEquals(1, stat.getBlockLocations().length);
}

13 Source : TestINodeFile.java
with Apache License 2.0
from NJUJYB

private static void checkEquals(RemoteIterator<LocatedFileStatus> i1, RemoteIterator<LocatedFileStatus> i2) throws IOException {
    while (i1.hasNext()) {
        replacedertTrue(i2.hasNext());
        // Compare all the fields but the path name, which is relative
        // to the original path from listFiles.
        LocatedFileStatus l1 = i1.next();
        LocatedFileStatus l2 = i2.next();
        replacedertEquals(l1.getAccessTime(), l2.getAccessTime());
        replacedertEquals(l1.getBlockSize(), l2.getBlockSize());
        replacedertEquals(l1.getGroup(), l2.getGroup());
        replacedertEquals(l1.getLen(), l2.getLen());
        replacedertEquals(l1.getModificationTime(), l2.getModificationTime());
        replacedertEquals(l1.getOwner(), l2.getOwner());
        replacedertEquals(l1.getPermission(), l2.getPermission());
        replacedertEquals(l1.getReplication(), l2.getReplication());
    }
    replacedertFalse(i2.hasNext());
}

13 Source : S3EventNotificationsPolicy.java
with Apache License 2.0
from mmolimar

@Override
public Iterator<FileMetadata> listFiles(FileSystem fs) {
    return sqs.receiveMessage(request).getMessages().stream().flatMap(message -> parseMessage(message).stream()).filter(record -> record.eventName.matches(eventNameRegex)).filter(record -> fs.getWorkingDirectory().toString().startsWith(getUriPrefix() + record.bucketName)).map(record -> {
        Path path = new Path(getUriPrefix() + record.bucketName + "/", record.objectKey);
        Optional<FileMetadata> metadata = Optional.empty();
        try {
            RemoteIterator<LocatedFileStatus> it = fs.listFiles(path, false);
            if (it.hasNext()) {
                LocatedFileStatus status = it.next();
                if (status.isFile())
                    metadata = Optional.of(toMetadata(status));
            }
        } catch (Exception ioe) {
            log.warn("{} Cannot get file at path '{}': {}", this, path, ioe.getMessage());
        }
        if (deleteMessages) {
            log.trace("{} Removing message with ID '{}'.", this, record.messageId);
            sqs.deleteMessage(queueUrl, record.receiptHandle);
        }
        return metadata;
    }).flatMap(metadataOpt -> metadataOpt.map(Stream::of).orElseGet(Stream::empty)).iterator();
}

13 Source : RollingSinkITCase.java
with Apache License 2.0
from ljygz

/**
 * This uses {@link org.apache.flink.streaming.connectors.fs.DateTimeBucketer} to
 * produce rolling files. The clock of DateTimeBucketer is set to
 * {@link ModifyableClock} to keep the time in lockstep with the processing of elements using
 * latches.
 */
@Test
public void testDateTimeRollingStringWriter() throws Exception {
    final int numElements = 20;
    final String outPath = hdfsURI + "/rolling-out";
    DateTimeBucketer.setClock(new ModifyableClock());
    ModifyableClock.setCurrentTime(0);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(2);
    DataStream<Tuple2<Integer, String>> source = env.addSource(new WaitingTestSourceFunction(numElements)).broadcast();
    // the parallel flatMap is chained to the sink, so when it has seen 5 elements it can
    // fire the latch
    DataStream<String> mapped = source.flatMap(new RichFlatMapFunction<Tuple2<Integer, String>, String>() {

        private static final long serialVersionUID = 1L;

        int count = 0;

        @Override
        public void flatMap(Tuple2<Integer, String> value, Collector<String> out) throws Exception {
            out.collect(value.f1);
            count++;
            if (count >= 5) {
                if (getRuntimeContext().getIndexOfThisSubtask() == 0) {
                    latch1.trigger();
                } else {
                    latch2.trigger();
                }
                count = 0;
            }
        }
    });
    RollingSink<String> sink = new RollingSink<String>(outPath).setBucketer(new DateTimeBucketer("ss")).setPartPrefix("part").setPendingPrefix("").setPendingSuffix("");
    mapped.addSink(sink);
    env.execute("RollingSink String Write Test");
    RemoteIterator<LocatedFileStatus> files = dfs.listFiles(new Path(outPath), true);
    // we should have 8 rolling files, 4 time intervals and parallelism of 2
    int numFiles = 0;
    while (files.hasNext()) {
        LocatedFileStatus file = files.next();
        numFiles++;
        if (file.getPath().toString().contains("rolling-out/00")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 0; i < 5; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/05")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 5; i < 10; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/10")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 10; i < 15; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else if (file.getPath().toString().contains("rolling-out/15")) {
            FSDataInputStream inStream = dfs.open(file.getPath());
            BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
            for (int i = 15; i < 20; i++) {
                String line = br.readLine();
                replacedert.replacedertEquals("message #" + i, line);
            }
            inStream.close();
        } else {
            replacedert.fail("File " + file + " does not match any expected roll pattern.");
        }
    }
    replacedert.replacedertEquals(8, numFiles);
}

13 Source : TestCompressDecompress.java
with Apache License 2.0
from Intel-bigdata

@Test
public void testListLocatedStatus() throws Exception {
    // if (!loadedNative()) {
    // return;
    // }
    waitTillSSMExitSafeMode();
    // initDB();
    SmartFileSystem smartDfs = new SmartFileSystem();
    smartDfs.initialize(dfs.getUri(), ssm.getContext().getConf());
    int arraySize = 1024 * 1024 * 8;
    String fileName = "/ssm/compression/file4";
    byte[] bytes = prepareFile(fileName, arraySize);
    // For uncompressed file, SmartFileSystem and DistributedFileSystem behave exactly the same
    RemoteIterator<LocatedFileStatus> iter1 = dfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat1 = iter1.next();
    RemoteIterator<LocatedFileStatus> iter2 = smartDfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat2 = iter2.next();
    replacedert.replacedertEquals(stat1.getPath(), stat2.getPath());
    replacedert.replacedertEquals(stat1.getBlockSize(), stat2.getBlockSize());
    replacedert.replacedertEquals(stat1.getLen(), stat2.getLen());
    BlockLocation[] blockLocations1 = stat1.getBlockLocations();
    BlockLocation[] blockLocations2 = stat2.getBlockLocations();
    replacedert.replacedertEquals(blockLocations1.length, blockLocations2.length);
    for (int i = 0; i < blockLocations1.length; i++) {
        replacedert.replacedertEquals(blockLocations1[i].getLength(), blockLocations2[i].getLength());
        replacedert.replacedertEquals(blockLocations1[i].getOffset(), blockLocations2[i].getOffset());
    }
    // Test compressed file
    int bufSize = 1024 * 1024;
    CmdletManager cmdletManager = ssm.getCmdletManager();
    long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
    waitTillActionDone(cmdId);
    RemoteIterator<LocatedFileStatus> iter3 = dfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat3 = iter3.next();
    BlockLocation[] blockLocations3 = stat3.getBlockLocations();
    RemoteIterator<LocatedFileStatus> iter4 = smartDfs.listLocatedStatus(new Path(fileName));
    LocatedFileStatus stat4 = iter4.next();
    BlockLocation[] blockLocations4 = stat4.getBlockLocations();
    replacedert.replacedertEquals(stat1.getPath(), stat4.getPath());
    replacedert.replacedertEquals(stat1.getBlockSize(), stat4.getBlockSize());
    replacedert.replacedertEquals(stat1.getLen(), stat4.getLen());
}

13 Source : IntegrationTestLoadAndVerify.java
with Apache License 2.0
from fengchen8086

static SortedSet<byte[]> readKeysToSearch(final Configuration conf) throws IOException, InterruptedException {
    Path keysInputDir = new Path(conf.get(SEARCHER_INPUTDIR_KEY));
    FileSystem fs = FileSystem.get(conf);
    SortedSet<byte[]> result = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
    if (!fs.exists(keysInputDir)) {
        throw new FileNotFoundException(keysInputDir.toString());
    }
    if (!fs.isDirectory(keysInputDir)) {
        FileStatus keyFileStatus = fs.getFileStatus(keysInputDir);
        readFileToSearch(conf, fs, keyFileStatus, result);
    } else {
        RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(keysInputDir, false);
        while (iterator.hasNext()) {
            LocatedFileStatus keyFileStatus = iterator.next();
            // Skip "_SUCCESS" file.
            if (keyFileStatus.getPath().getName().startsWith("_"))
                continue;
            readFileToSearch(conf, fs, keyFileStatus, result);
        }
    }
    return result;
}

13 Source : FileComponentsSource.java
with GNU General Public License v3.0
from dlanza1

public Properties loadProperties() throws Exception {
    Properties props = new Properties();
    RemoteIterator<LocatedFileStatus> fileStatusLisreplacederator = fs.listFiles(new Path(path), true);
    while (fileStatusLisreplacederator.hasNext()) {
        LocatedFileStatus fileStatus = fileStatusLisreplacederator.next();
        if (fileStatus.isFile()) {
            String filePath = fileStatus.getPath().toString();
            LOG.info("Reading " + filePath);
            try {
                Properties fileProps = Properties.fromFile(filePath);
                props.putAll(fileProps);
                LOG.info("Loaded " + filePath);
            } catch (Exception e) {
                LOG.error(e.getMessage(), e);
            }
        }
    }
    return props;
}

See More Examples