Here are the examples of the java api org.apache.hadoop.fs.LocatedFileStatus taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
60 Examples
19
Source : HistoryLogUtilsTest.java
with Apache License 2.0
from spotify
with Apache License 2.0
from spotify
private LocatedFileStatus mockFileStatus(String filePath) {
LocatedFileStatus mockLFS = Mockito.mock(LocatedFileStatus.clreplaced);
Mockito.when(mockLFS.getPath()).thenReturn(new Path(filePath));
return mockLFS;
}
19
Source : ViewFsLocatedFileStatus.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
clreplaced ViewFsLocatedFileStatus extends LocatedFileStatus {
final LocatedFileStatus myFs;
Path modifiedPath;
ViewFsLocatedFileStatus(LocatedFileStatus locatedFileStatus, Path path) {
myFs = locatedFileStatus;
modifiedPath = path;
}
@Override
public long getLen() {
return myFs.getLen();
}
@Override
public boolean isFile() {
return myFs.isFile();
}
@Override
public boolean isDirectory() {
return myFs.isDirectory();
}
@Override
@SuppressWarnings("deprecation")
public boolean isDir() {
return myFs.isDirectory();
}
@Override
public boolean isSymlink() {
return myFs.isSymlink();
}
@Override
public long getBlockSize() {
return myFs.getBlockSize();
}
@Override
public short getReplication() {
return myFs.getReplication();
}
@Override
public long getModificationTime() {
return myFs.getModificationTime();
}
@Override
public long getAccessTime() {
return myFs.getAccessTime();
}
@Override
public FsPermission getPermission() {
return myFs.getPermission();
}
@Override
public String getOwner() {
return myFs.getOwner();
}
@Override
public String getGroup() {
return myFs.getGroup();
}
@Override
public Path getPath() {
return modifiedPath;
}
@Override
public void setPath(final Path p) {
modifiedPath = p;
}
@Override
public Path getSymlink() throws IOException {
return myFs.getSymlink();
}
@Override
public void setSymlink(Path p) {
myFs.setSymlink(p);
}
@Override
public BlockLocation[] getBlockLocations() {
return myFs.getBlockLocations();
}
@Override
public int compareTo(Object o) {
return super.compareTo(o);
}
@Override
public boolean equals(Object o) {
return super.equals(o);
}
@Override
public int hashCode() {
return super.hashCode();
}
}
19
Source : FSAgent.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
public List<String> listFiles(String dir) throws IOException {
List<String> files = new ArrayList<>();
Path path = new Path(dir);
FileStatus fileStatus = fileSystem.getFileStatus(path);
if (!fileStatus.isDirectory()) {
throw new FileNotFoundException("Cannot read directory " + dir);
}
RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
while (it.hasNext()) {
LocatedFileStatus lfs = it.next();
files.add(lfs.getPath().getName());
}
return files;
}
19
Source : FSAgent.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
public List<LocatedFileStatus> listFilesInfo(String dir) throws IOException {
List<LocatedFileStatus> files = new ArrayList<>();
Path path = new Path(dir);
FileStatus fileStatus = fileSystem.getFileStatus(path);
if (!fileStatus.isDirectory()) {
throw new FileNotFoundException("Cannot read directory " + dir);
}
RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
while (it.hasNext()) {
LocatedFileStatus lfs = it.next();
files.add(lfs);
}
return files;
}
19
Source : HDFSEasy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public List<LocatedFileStatus> listFiles(Path path) throws IOException {
RemoteIterator<LocatedFileStatus> i = dfs.listFiles(path, false);
List<LocatedFileStatus> retList = new ArrayList<>();
while (i.hasNext()) {
LocatedFileStatus locatedFileStatus = i.next();
retList.add(locatedFileStatus);
}
return retList;
}
18
Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, int bucketNumber, Optional<DeleteDeltaLocations> deleteDeltaLocations) {
return createInternalHiveSplit(status, OptionalInt.of(bucketNumber), false, deleteDeltaLocations, Optional.empty());
}
18
Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, boolean splittable, Optional<DeleteDeltaLocations> deleteDeltaLocations, Optional<Long> startRowOffsetOfFile) {
return createInternalHiveSplit(status, OptionalInt.empty(), splittable, deleteDeltaLocations, startRowOffsetOfFile);
}
18
Source : GenerateData.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
static DataStatistics publishPlainDataStatistics(Configuration conf, Path inputDir) throws IOException {
FileSystem fs = inputDir.getFileSystem(conf);
// obtain input data file statuses
long dataSize = 0;
long fileCount = 0;
RemoteIterator<LocatedFileStatus> iter = fs.listFiles(inputDir, true);
PathFilter filter = new Utils.OutputFileUtils.OutputFilesFilter();
while (iter.hasNext()) {
LocatedFileStatus lStatus = iter.next();
if (filter.accept(lStatus.getPath())) {
dataSize += lStatus.getLen();
++fileCount;
}
}
// publish the plain data statistics
LOG.info("Total size of input data : " + StringUtils.humanReadableInt(dataSize));
LOG.info("Total number of input data files : " + fileCount);
return new DataStatistics(dataSize, fileCount, false);
}
18
Source : HDFSTest.java
with Apache License 2.0
from junneyang
with Apache License 2.0
from junneyang
@Test
public void testList() throws FileNotFoundException, IOException {
Path f = new Path("/");
RemoteIterator<LocatedFileStatus> files = fs.listFiles(f, true);
while (files.hasNext()) {
LocatedFileStatus file = (LocatedFileStatus) files.next();
LOGGER.info("====={}", file.getPath());
}
}
18
Source : CamelSinkHDFSITCase.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void printFile(LocatedFileStatus f, String matchString) {
try {
String contents = hdfsEasy.readFile(f.getPath());
LOG.debug("Retrieved file {} with contents: {}", f.getPath(), contents);
boolean contains = contents.contains(matchString);
replacedertTrue(contains, "Unexpected content for the remote file " + f.getPath().getName() + " content: [" + contents + "] should contain [" + matchString + "]");
} catch (IOException e) {
LOG.debug("Reading returned file {} failed: {}", f.getPath(), e.getMessage());
fail("I/O error: " + e.getMessage());
}
}
17
Source : HiveFileIterator.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
@Override
protected LocatedFileStatus computeNext() {
while (true) {
while (remoteIterator.hasNext()) {
LocatedFileStatus status = getLocatedFileStatus(remoteIterator);
// Ignore hidden files and directories. Hive ignores files starting with _ and . as well.
String fileName = status.getPath().getName();
if (fileName.startsWith("_") || fileName.startsWith(".")) {
continue;
}
if (status.isDirectory()) {
switch(nestedDirectoryPolicy) {
case IGNORED:
continue;
case RECURSE:
paths.add(status.getPath());
continue;
case FAIL:
throw new NestedDirectoryNotAllowedException(status.getPath());
}
}
return status;
}
if (paths.isEmpty()) {
return endOfData();
}
remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst());
}
}
17
Source : Cloudup.java
with Apache License 2.0
from steveloughran
with Apache License 2.0
from steveloughran
/**
* List the source files and build the list.
* @return list of uploads
* @throws IOException failure to list
*/
private List<UploadEntry> createUploadList() throws IOException {
List<UploadEntry> uploads = new ArrayList<>();
RemoteIterator<LocatedFileStatus> ri = sourceFS.listFiles(sourcePath, true);
while (ri.hasNext()) {
LocatedFileStatus status = ri.next();
UploadEntry entry = new UploadEntry(status);
entry.setDest(getFinalPath(status.getPath()));
uploads.add(entry);
}
LOG.info("List {}", ri);
if (ri instanceof Closeable) {
((Closeable) ri).close();
}
return uploads;
}
17
Source : SegmentHelper.java
with Apache License 2.0
from shunfei
with Apache License 2.0
from shunfei
public static void literalAllSegments(FileSystem fileSystem, Path dir, Consumer<LocatedFileStatus> consumer) throws IOException {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(dir, true);
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
if (!fileStatus.isFile()) {
continue;
}
if (fileStatus.getLen() == 0) {
continue;
}
Path path = fileStatus.getPath();
if (checkSegmentByPath(path)) {
consumer.accept(fileStatus);
}
}
}
17
Source : HiveFileIterator.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
@Override
protected LocatedFileStatus computeNext() {
while (true) {
while (remoteIterator.hasNext()) {
LocatedFileStatus status = getLocatedFileStatus(remoteIterator);
// Ignore hidden files and directories. Hive ignores files starting with _ and . as well.
String fileName = status.getPath().getName();
if (fileName.startsWith("_") || fileName.startsWith(".")) {
continue;
}
if (status.isDirectory()) {
switch(nestedDirectoryPolicy) {
case IGNORED:
continue;
case RECURSE:
paths.add(status.getPath());
continue;
case FAIL:
throw new NestedDirectoryNotAllowedException();
}
}
return status;
}
if (paths.isEmpty()) {
return endOfData();
}
remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst());
}
}
17
Source : TestV2LsOperations.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* To get this project to compile under Hadoop 1, this code needs to be
* commented out
*
* @param fs filesystem
* @param dir dir
* @param subdir subdir
* @param recursive recurse?
* @throws IOException IO problems
*/
public static void replacedertListFilesFinds(FileSystem fs, Path dir, Path subdir, boolean recursive) throws IOException {
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(dir, recursive);
boolean found = false;
int entries = 0;
StringBuilder builder = new StringBuilder();
while (iterator.hasNext()) {
LocatedFileStatus next = iterator.next();
entries++;
builder.append(next.toString()).append('\n');
if (next.getPath().equals(subdir)) {
found = true;
}
}
replacedertTrue("Path " + subdir + " not found in directory " + dir + " : " + " entries=" + entries + " content" + builder.toString(), found);
}
17
Source : FilePaths.java
with Apache License 2.0
from ExpediaInceCommercePlatform
with Apache License 2.0
from ExpediaInceCommercePlatform
/**
* Adds a file to the object
*
* @param file {@link LocatedFileStatus}
*/
public void addFile(final LocatedFileStatus file) {
bytes += file.getLen();
pathsByBytes.put(file.getPath(), file.getLen());
}
16
Source : FileSegmentPool.java
with Apache License 2.0
from shunfei
with Apache License 2.0
from shunfei
public void refreshLocalities() {
try {
// HashMap taks muti-thread risk here. Change to ConcurrentHashMap if it happens.
Map<String, List<String>> newHostMap = new HashMap<>(segmentFdMap.size());
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(segmentRootPath, true);
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
if (fileStatus.getLen() == 0) {
continue;
}
String name = getSegmentName(fileStatus);
if (name == null) {
continue;
}
BlockLocation[] locations = fileStatus.getBlockLocations();
if (locations.length != 1) {
logger.error("A segment should only consisted by one block, now {}. Ignored: {}", locations.length, name);
continue;
}
List<String> hosts = Arrays.asList(locations[0].getHosts());
newHostMap.put(name, hosts);
}
hostMap = newHostMap;
} catch (IOException e) {
if (e instanceof ClosedByInterruptException) {
logger.warn("Refresh [{}] segment locality failed by ClosedByInterruptException.", tableName);
// Normally close interrupt.
return;
}
String msg = e.getMessage();
if (msg != null && Strings.equals(msg.trim(), "Filesystem closed")) {
logger.warn("Refresh [{}] segment locality failed by Filesystem closed.", tableName);
// Normally close interrupt.
return;
}
logger.warn("Refresh [{}] segment locality failed.", tableName, e);
}
}
16
Source : AbstractFlagConfig.java
with Apache License 2.0
from NationalSecurityAgency
with Apache License 2.0
from NationalSecurityAgency
protected Path getTestFile(FileSystem fs) throws IOException {
createTestFiles(1, 1);
Path file = null;
for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(this.fmc.getBaseHDFSDir()), true); it.hasNext(); ) {
LocatedFileStatus status = it.next();
if (status.isFile()) {
file = status.getPath();
break;
}
}
return file;
}
16
Source : RecordingsAgent.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
public List<RecordingInfo> getRecordingInfo(String appId) {
List<RecordingInfo> result = new ArrayList<>();
String dir = getRecordingsDirectory(appId);
if (dir == null) {
return result;
}
Path path = new Path(dir);
try {
FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
if (!fileStatus.isDirectory()) {
return result;
}
RemoteIterator<LocatedFileStatus> ri = stramAgent.getFileSystem().listLocatedStatus(path);
while (ri.hasNext()) {
LocatedFileStatus lfs = ri.next();
if (lfs.isDirectory()) {
try {
String opId = lfs.getPath().getName();
result.addAll(getRecordingInfo(appId, opId));
} catch (NumberFormatException ex) {
// ignore
}
}
}
} catch (IOException ex) {
LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
return result;
}
return result;
}
16
Source : RecordingsAgent.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
private List<RecordingInfo> getRecordingInfoHelper(String appId, String opId, Set<String> containers) {
List<RecordingInfo> result = new ArrayList<>();
String dir = getRecordingsDirectory(appId, opId);
if (dir == null) {
return result;
}
Path path = new Path(dir);
try {
FileStatus fileStatus = stramAgent.getFileSystem().getFileStatus(path);
if (!fileStatus.isDirectory()) {
return result;
}
RemoteIterator<LocatedFileStatus> ri = stramAgent.getFileSystem().listLocatedStatus(path);
while (ri.hasNext()) {
LocatedFileStatus lfs = ri.next();
if (lfs.isDirectory()) {
try {
String id = lfs.getPath().getName();
RecordingInfo recordingInfo = getRecordingInfoHelper(appId, opId, id, containers);
if (recordingInfo != null) {
result.add(recordingInfo);
}
} catch (NumberFormatException ex) {
// ignore
}
}
}
} catch (IOException ex) {
LOG.warn("Cannot get recording info for app id {}: {}", appId, ex);
return result;
}
return result;
}
16
Source : HadoopFileSystemUtils.java
with GNU General Public License v3.0
from icgc-dcc
with GNU General Public License v3.0
from icgc-dcc
private static List<LocatedFileStatus> getFiles(FileSystem fileSystem, Path target, boolean recusre) {
val results = Lists.<LocatedFileStatus>newArrayList();
RemoteIterator<LocatedFileStatus> fileStatusLisreplacederator = null;
try {
fileStatusLisreplacederator = fileSystem.listFiles(target, true);
while (fileStatusLisreplacederator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusLisreplacederator.next();
results.add(fileStatus);
}
} catch (IOException e) {
log.info("Error retriving files in path '{}'", target);
}
return results;
}
16
Source : TestIcebergStreamWriter.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private Set<String> scanDataFiles() throws IOException {
Path dataDir = new Path(tablePath, "data");
FileSystem fs = FileSystem.get(new Configuration());
if (!fs.exists(dataDir)) {
return ImmutableSet.of();
} else {
Set<String> paths = Sets.newHashSet();
RemoteIterator<LocatedFileStatus> iterators = fs.listFiles(dataDir, true);
while (iterators.hasNext()) {
LocatedFileStatus status = iterators.next();
if (status.isFile()) {
Path path = status.getPath();
if (path.getName().endsWith("." + format.toString().toLowerCase())) {
paths.add(path.toString());
}
}
}
return paths;
}
}
15
Source : HdfsIOBenchmark.java
with Apache License 2.0
from zrlio
with Apache License 2.0
from zrlio
void browseDir() throws Exception {
System.out.println("reading enumarate dir, path " + path);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
// benchmark
System.out.println("starting benchmark...");
RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, false);
while (iter.hasNext()) {
LocatedFileStatus status = iter.next();
System.out.println(status.getPath());
}
fs.close();
}
15
Source : InternalHiveSplitFactory.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
public Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable, Optional<AcidInfo> acidInfo) {
splittable = splittable && status.getLen() > minimumTargetSplitSizeInBytes && isSplittable(inputFormat, fileSystem, status.getPath());
return createInternalHiveSplit(status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), status.getModificationTime(), bucketNumber, splittable, acidInfo);
}
15
Source : ITestLocalCloudup.java
with Apache License 2.0
from steveloughran
with Apache License 2.0
from steveloughran
@Test
public void testCopyRecursive() throws Throwable {
int expected = createTestFiles(sourceDir, 64);
expectSuccess(new Cloudup(), "-s", sourceDir.toURI().toString(), "-d", destDir.toURI().toString(), "-t", "4", "-l", "3");
LocalFileSystem local = FileSystem.getLocal(new Configuration());
Set<String> entries = new TreeSet<>();
RemoteIterator<LocatedFileStatus> iterator = local.listFiles(new Path(destDir.toURI()), true);
int count = 0;
while (iterator.hasNext()) {
LocatedFileStatus next = iterator.next();
entries.add(next.getPath().toUri().toString());
LOG.info("Entry {} size = {}", next.getPath(), next.getLen());
count++;
}
replacedertEquals("Mismatch in files found", expected, count);
}
15
Source : ApplicationDriver.java
with GNU Lesser General Public License v3.0
from schic
with GNU Lesser General Public License v3.0
from schic
private List<String> buildJarFiles(final MutableRef<String> primaryJarRef) throws IOException {
final List<String> list = new ArrayList<>();
final Path directoryPath = new Path(_jarDirectoryPath);
final RemoteIterator<LocatedFileStatus> files = _fileSystem.listFiles(directoryPath, false);
while (files.hasNext()) {
final LocatedFileStatus file = files.next();
final Path path = file.getPath();
final String filename = path.getName();
boolean primaryJar = false;
for (final String prefix : PRIMARY_JAR_FILENAME_PREFIXES) {
if (filename.startsWith(prefix)) {
primaryJarRef.set(path.toString());
primaryJar = true;
break;
}
}
if (!primaryJar) {
list.add(path.toString());
}
}
if (primaryJarRef.get() == null) {
throw new IllegalArgumentException("Failed to find primary jar (starting with '" + PRIMARY_JAR_FILENAME_PREFIXES[0] + "') in JAR file directory: " + _jarDirectoryPath);
}
return list;
}
15
Source : InternalHiveSplitFactory.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
private Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable, Optional<DeleteDeltaLocations> deleteDeltaLocations, Optional<Long> startRowOffsetOfFile) {
splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath());
return createInternalHiveSplit(status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), status.getModificationTime(), bucketNumber, splittable, deleteDeltaLocations, startRowOffsetOfFile);
}
15
Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Test when input patch has a symbolic links as its children
*/
@Test
public void testSymbolicLinks() throws IOException {
writeFile(fc, FILE1, FILE_LEN);
writeFile(fc, FILE2, FILE_LEN);
writeFile(fc, FILE3, FILE_LEN);
Path dir4 = new Path(TEST_DIR, "dir4");
Path dir5 = new Path(dir4, "dir5");
Path file4 = new Path(dir4, "file4");
fc.createSymlink(DIR1, dir5, true);
fc.createSymlink(FILE1, file4, true);
RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(dir4, true);
LocatedFileStatus stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE3), stat.getPath());
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertFalse(itor.hasNext());
itor = fc.util().listFiles(dir4, false);
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertFalse(itor.hasNext());
}
15
Source : QueryInputFormat.java
with Apache License 2.0
from Merck
with Apache License 2.0
from Merck
public static void addQueryRecursively(Configuration conf, Path path, boolean sparqlUpdate, int stage) throws IOException {
RemoteIterator<LocatedFileStatus> iter = path.getFileSystem(conf).listLocatedStatus(path);
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
if (stat.isDirectory()) {
addQueryRecursively(conf, stat.getPath(), sparqlUpdate, stage);
} else {
addQuery(conf, stat, sparqlUpdate, stage);
}
}
}
15
Source : ColumnarFilesReader.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
void checkPath() {
try {
RemoteIterator<LocatedFileStatus> files = fs.listFiles(folderPath, false);
if (files == null) {
throw new IllegalArgumentException("Invalid path " + folderPath);
}
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
Path path = fileStatus.getPath();
String name = path.getName();
if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
dataFilePath = path;
} else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
metaFilePath = path;
} else {
logger.warn("Contains invalid file {} in path {}", path, folderPath);
}
}
if (dataFilePath == null || metaFilePath == null) {
throw new IllegalArgumentException("Invalid path " + folderPath);
}
} catch (IOException e) {
throw new RuntimeException("io error", e);
}
}
15
Source : HDFSResourceStore.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
@Override
protected void visitFolderImpl(String folderPath, boolean recursive, VisitFilter filter, boolean loadContent, Visitor visitor) throws IOException {
Path p = getRealHDFSPath(folderPath);
if (!fs.exists(p) || !fs.isDirectory(p)) {
return;
}
String fsPathPrefix = p.toUri().getPath();
String resPathPrefix = folderPath.endsWith("/") ? folderPath : folderPath + "/";
RemoteIterator<LocatedFileStatus> it = fs.listFiles(p, recursive);
while (it.hasNext()) {
LocatedFileStatus status = it.next();
if (status.isDirectory())
continue;
String path = status.getPath().toUri().getPath();
if (!path.startsWith(fsPathPrefix))
throw new IllegalStateException("File path " + path + " is supposed to start with " + fsPathPrefix);
String resPath = resPathPrefix + path.substring(fsPathPrefix.length() + 1);
if (filter.matches(resPath, status.getModificationTime())) {
RawResource raw;
if (loadContent)
raw = new RawResource(resPath, status.getModificationTime(), fs.open(status.getPath()));
else
raw = new RawResource(resPath, status.getModificationTime());
try {
visitor.visit(raw);
} finally {
raw.close();
}
}
}
}
15
Source : HadoopFileSystem.java
with Apache License 2.0
from DSC-SPIDAL
with Apache License 2.0
from DSC-SPIDAL
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
*
* @param f given path
* @return the statuses of the files/directories in the given patch
*/
@Override
public FileStatus[] listFiles(Path f) throws IOException {
RemoteIterator<LocatedFileStatus> listFiles = this.hadoopFileSystem.listFiles(toHadoopPath(f), true);
List<FileStatus> statusList = new ArrayList<>();
while (listFiles.hasNext()) {
LocatedFileStatus next = listFiles.next();
FileStatus status = new HadoopFileStatus(next);
statusList.add(status);
}
return statusList.toArray(new FileStatus[0]);
}
15
Source : TestRemoteNodeFileSystemDual.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
@Test
public void testClientWriteEmptyFile() throws Exception {
Path basePath = new Path(temporaryFolder.newFolder().getAbsolutePath());
Path path = ((PathCanonicalizer) clientFS).canonicalizePath(new Path(basePath, "testfile.bytes"));
// create a file
FSDataOutputStream stream = clientFS.create(path, false);
// close it without writing anything to it
stream.close();
// make sure the file was created
RemoteIterator<LocatedFileStatus> iter = client.fileSystem.listFiles(basePath, false);
replacedertEquals(true, iter.hasNext());
LocatedFileStatus status = iter.next();
try (FSDataInputStream in = clientFS.open(status.getPath())) {
in.readByte();
fail("Fail is expected to be empty");
} catch (EOFException e) {
// empty file as expected
}
client.fileSystem.delete(status.getPath(), false);
}
14
Source : FileInputFormat.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Add files in the input path recursively into the results.
* @param result
* The List to store all files.
* @param fs
* The FileSystem.
* @param path
* The input path.
* @param inputFilter
* The input filter that can be used to filter files/dirs.
* @throws IOException
*/
protected void addInputPathRecursively(List<FileStatus> result, FileSystem fs, Path path, PathFilter inputFilter) throws IOException {
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
while (iter.hasNext()) {
LocatedFileStatus stat = iter.next();
if (inputFilter.accept(stat.getPath())) {
if (stat.isDirectory()) {
addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
} else {
result.add(stat);
}
}
}
}
14
Source : BucketingSinkTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This uses {@link DateTimeBucketer} to
* produce rolling files. We use {@link OneInputStreamOperatorTestHarness} to manually
* advance processing time.
*/
@Test
public void testDateTimeRollingStringWriter() throws Exception {
final int numElements = 20;
final String outPath = hdfsURI + "/rolling-out";
BucketingSink<String> sink = new BucketingSink<String>(outPath).setBucketer(new DateTimeBucketer<String>("ss")).setPartPrefix(PART_PREFIX).setPendingPrefix("").setPendingSuffix("");
OneInputStreamOperatorTestHarness<String, Object> testHarness = createTestSink(sink, 1, 0);
testHarness.setProcessingTime(0L);
testHarness.setup();
testHarness.open();
for (int i = 0; i < numElements; i++) {
// Every 5 elements, increase the clock time. We should end up with 5 elements per bucket.
if (i % 5 == 0) {
testHarness.setProcessingTime(i * 1000L);
}
testHarness.processElement(new StreamRecord<>("message #" + Integer.toString(i)));
}
testHarness.close();
RemoteIterator<LocatedFileStatus> files = dfs.listFiles(new Path(outPath), true);
// We should have 4 rolling files across 4 time intervals
int numFiles = 0;
while (files.hasNext()) {
LocatedFileStatus file = files.next();
numFiles++;
if (file.getPath().toString().contains("rolling-out/00")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 0; i < 5; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/05")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 5; i < 10; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/10")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 10; i < 15; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/15")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 15; i < 20; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else {
replacedert.fail("File " + file + " does not match any expected roll pattern.");
}
}
replacedert.replacedertEquals(4, numFiles);
}
14
Source : ColumnarSplitInputFormat.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
private boolean isValidFragmentPath(FileSystem fs, Path path) throws IOException {
RemoteIterator<LocatedFileStatus> files = fs.listFiles(path, false);
if (files == null) {
logger.warn("Invalid fragment path:{}, empty folder", path);
return false;
}
boolean hasDataFile = false;
boolean hasMetaFile = false;
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
Path childPath = fileStatus.getPath();
String name = childPath.getName();
if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
hasDataFile = true;
} else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
hasMetaFile = true;
} else {
logger.warn("Contains invalid file {} in path {}", childPath, path);
}
}
if (hasDataFile && hasMetaFile) {
return true;
} else {
logger.warn("Invalid fragment path:{}, data file exists:{}, meta file exists:{}", path, hasDataFile, hasMetaFile);
return false;
}
}
14
Source : TestRemoteNodeFileSystemDual.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
@Test
public void basicClientReadWrite() throws Exception {
Path basePath = new Path(temporaryFolder.newFolder().getAbsolutePath());
Path path = ((PathCanonicalizer) clientFS).canonicalizePath(new Path(basePath, "testfile.bytes"));
final byte[] randomBytesMoreThanBuffer = new byte[RemoteNodeFileSystem.REMOTE_WRITE_BUFFER_SIZE_DEFAULT * 3];
Random r = new Random();
r.nextBytes(randomBytesMoreThanBuffer);
try (FSDataOutputStream stream = clientFS.create(path, false)) {
stream.write(randomBytesMoreThanBuffer);
}
RemoteIterator<LocatedFileStatus> iter = client.fileSystem.listFiles(basePath, false);
replacedertEquals(true, iter.hasNext());
LocatedFileStatus status = iter.next();
try (FSDataInputStream in = clientFS.open(status.getPath())) {
byte[] back = new byte[randomBytesMoreThanBuffer.length];
int dataRead = in.read(back);
replacedertEquals(back.length, dataRead);
replacedertTrue(Arrays.equals(randomBytesMoreThanBuffer, back));
}
client.fileSystem.delete(status.getPath(), false);
}
14
Source : ProtoParquetWriterWithOffsetTest.java
with Apache License 2.0
from criteo
with Apache License 2.0
from criteo
private List<EventHeaderProtos.Header> checkSingleFileWithFileSystem(Collection<EventHeaderProtos.Header> inputHeaders) throws IOException {
final List<EventHeaderProtos.Header> headers = new LinkedList<>();
Path newTmpFile = new Path(tmpPath, "file");
final ProtoParquetWriter<Message> writer = new ProtoParquetWriter<>(newTmpFile, EventHeaderProtos.Header.clreplaced);
long offset = 1;
final BiConsumer<String, String> protoMetadataWriter = mock(BiConsumer.clreplaced);
final ProtoParquetWriterWithOffset consumer = new ProtoParquetWriterWithOffset<>(writer, newTmpFile, finalPath, localFs, new FixedOffsetComputer(FINAL_FILE_NAME, 123), UTC_EPOCH, "ignored", protoMetadataWriter, 1);
for (EventHeaderProtos.Header header : inputHeaders) {
consumer.write(1234567890L, header, new TopicParreplacedionOffset(TOPIC, 1, offset++));
}
consumer.close();
final RemoteIterator<LocatedFileStatus> filesIterator = localFs.listFiles(finalPath, false);
final LocatedFileStatus fileStatus = filesIterator.next();
replacedert.replacedertEquals(FINAL_FILE_NAME, fileStatus.getPath().getName());
replacedert.replacedertFalse("There should be only one output file", filesIterator.hasNext());
final ParquetReader<EventHeaderProtos.Header.Builder> reader;
reader = ProtoParquetReader.<EventHeaderProtos.Header.Builder>builder(fileStatus.getPath()).build();
EventHeaderProtos.Header.Builder current = reader.read();
while (current != null) {
headers.add(current.build());
current = reader.read();
}
return headers;
}
14
Source : TestOzoneFileInterfaces.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Test
public void testOzoneManagerLocatedFileStatus() throws IOException {
String data = RandomStringUtils.randomAlphanumeric(20);
String filePath = RandomStringUtils.randomAlphanumeric(5);
Path path = createPath("/" + filePath);
try (FSDataOutputStream stream = fs.create(path)) {
stream.writeBytes(data);
}
FileStatus status = fs.getFileStatus(path);
replacedertTrue(status instanceof LocatedFileStatus);
LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
replacedertTrue(locatedFileStatus.getBlockLocations().length >= 1);
for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) {
replacedertTrue(blockLocation.getNames().length >= 1);
replacedertTrue(blockLocation.getHosts().length >= 1);
}
}
14
Source : TestOzoneFileInterfaces.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Test
@Ignore("HDDS-3506")
public void testOzoneManagerLocatedFileStatusBlockOffsetsWithMultiBlockFile() throws Exception {
// naive replacedumption: MiniOzoneCluster will not have larger than ~1GB
// block size when running this test.
int blockSize = (int) fs.getConf().getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
String data = RandomStringUtils.randomAlphanumeric(2 * blockSize + 837);
String filePath = RandomStringUtils.randomAlphanumeric(5);
Path path = createPath("/" + filePath);
try (FSDataOutputStream stream = fs.create(path)) {
stream.writeBytes(data);
}
FileStatus status = fs.getFileStatus(path);
replacedertTrue(status instanceof LocatedFileStatus);
LocatedFileStatus locatedFileStatus = (LocatedFileStatus) status;
BlockLocation[] blockLocations = locatedFileStatus.getBlockLocations();
replacedertEquals(0, blockLocations[0].getOffset());
replacedertEquals(blockSize, blockLocations[1].getOffset());
replacedertEquals(2 * blockSize, blockLocations[2].getOffset());
replacedertEquals(blockSize, blockLocations[0].getLength());
replacedertEquals(blockSize, blockLocations[1].getLength());
replacedertEquals(837, blockLocations[2].getLength());
}
13
Source : TestCTTAS.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
private List<Path> findTemporaryTableLocation(String tableName) throws IOException {
Path sessionTempLocation = new Path(getDfsTestTmpSchemaLocation(), session_id.toString());
replacedertTrue("Session temporary location must exist", fs.exists(sessionTempLocation));
replacedertEquals("Session temporary location permission should match", expectedFolderPermission, fs.getFileStatus(sessionTempLocation).getPermission());
String tableUUID = UUID.nameUUIDFromBytes(tableName.getBytes()).toString();
RemoteIterator<LocatedFileStatus> pathList = fs.listLocatedStatus(sessionTempLocation);
List<Path> matchingPath = Lists.newArrayList();
while (pathList.hasNext()) {
LocatedFileStatus path = pathList.next();
if (path.isDirectory() && path.getPath().getName().equals(tableUUID)) {
matchingPath.add(path.getPath());
}
}
return matchingPath;
}
13
Source : FileSegmentPool.java
with Apache License 2.0
from shunfei
with Apache License 2.0
from shunfei
private synchronized boolean doRefreshSegments() throws IOException {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(segmentRootPath, true);
Set<String> nameSet = new HashSet<>(2048);
int updateCount = 0;
boolean hasError = false;
// Load segment from storage.
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
String name = getSegmentName(fileStatus);
if (fileStatus.getLen() == 0 || name == null) {
continue;
}
nameSet.add(name);
long modifyTime = fileStatus.getModificationTime();
SegmentFdAndTime st = segmentFdMap.get(name);
if (st != null && st.modifyTime == modifyTime && st.fileSize == fileStatus.getLen()) {
// Segment exist and is up to dated.
continue;
}
BlockLocation[] locations = fileStatus.getBlockLocations();
if (locations.length != 1) {
logger.error("A segment should only consisted by one block, now {}. Ignored: {}", locations.length, name);
continue;
}
Path segmentPath = segmentPath(name);
ByteBufferReader.Opener readerOpener = ByteBufferReader.Opener.create(fileSystem, segmentPath, fileStatus.getLen(), locations.length);
SegmentMeta sectionInfo = null;
try (ByteBufferReader reader = readerOpener.open(0)) {
sectionInfo = Integrate.INSTANCE.read(reader);
if (sectionInfo == null) {
// Not a segment.
continue;
}
IntegratedSegment.Fd fd = IntegratedSegment.Fd.create(name, sectionInfo, readerOpener);
if (fd.info().rowCount() == 0) {
logger.debug("table [{}] ignore empty segment [{}]", tableName, name);
} else {
segmentFdMap.put(name, new SegmentFdAndTime(fd, fileStatus.getModificationTime(), fileStatus.getLen()));
logger.info("table [{}] add new segment [{}]", tableName, name);
updateCount++;
}
} catch (IOException e) {
hasError = true;
logger.error("", e);
logger.error("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
logger.error("Load segment [{}: {}] failed, system in inconsistent state", tableName, name);
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
}
}
// Remove outdated segments.
Iterator<String> it = segmentFdMap.keySet().iterator();
while (it.hasNext()) {
String name = it.next();
if (!nameSet.contains(name)) {
logger.info("table [{}] remove segment [{}]", tableName, name);
it.remove();
updateCount++;
}
}
if (updateCount > 0) {
updateSegmentFdList();
}
// Save to local cache.
totalUpdateCount += updateCount;
if (totalUpdateCount >= SaveCacheUpdateCount || lastSaveCacheTime + SaveCachePeriod <= System.currentTimeMillis() || mustRefresh) {
Try.on(this::saveToLocalCache, 1, logger, String.format("Save %s segment fds to local cache failed", tableName));
logger.debug("save [{}] segment fds to local cache", tableName);
totalUpdateCount = 0;
lastSaveCacheTime = System.currentTimeMillis();
}
mustRefresh = false;
return !hasError;
}
13
Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Test when input path is a directory
*/
@Test
public void testDirectory() throws IOException {
fc.mkdir(DIR1, FsPermission.getDefault(), true);
// test empty directory
RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(DIR1, true);
replacedertFalse(itor.hasNext());
itor = fc.util().listFiles(DIR1, false);
replacedertFalse(itor.hasNext());
// testing directory with 1 file
writeFile(fc, FILE2, FILE_LEN);
itor = fc.util().listFiles(DIR1, true);
LocatedFileStatus stat = itor.next();
replacedertFalse(itor.hasNext());
replacedertTrue(stat.isFile());
replacedertEquals(FILE_LEN, stat.getLen());
replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
replacedertEquals(1, stat.getBlockLocations().length);
itor = fc.util().listFiles(DIR1, false);
stat = itor.next();
replacedertFalse(itor.hasNext());
replacedertTrue(stat.isFile());
replacedertEquals(FILE_LEN, stat.getLen());
replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
replacedertEquals(1, stat.getBlockLocations().length);
// test more complicated directory
writeFile(fc, FILE1, FILE_LEN);
writeFile(fc, FILE3, FILE_LEN);
itor = fc.util().listFiles(TEST_DIR, true);
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE2), stat.getPath());
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE3), stat.getPath());
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertFalse(itor.hasNext());
itor = fc.util().listFiles(TEST_DIR, false);
stat = itor.next();
replacedertTrue(stat.isFile());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertFalse(itor.hasNext());
}
13
Source : TestListFilesInFileContext.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Test when input path is a file
*/
@Test
public void testFile() throws IOException {
fc.mkdir(TEST_DIR, FsPermission.getDefault(), true);
writeFile(fc, FILE1, FILE_LEN);
RemoteIterator<LocatedFileStatus> itor = fc.util().listFiles(FILE1, true);
LocatedFileStatus stat = itor.next();
replacedertFalse(itor.hasNext());
replacedertTrue(stat.isFile());
replacedertEquals(FILE_LEN, stat.getLen());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertEquals(1, stat.getBlockLocations().length);
itor = fc.util().listFiles(FILE1, false);
stat = itor.next();
replacedertFalse(itor.hasNext());
replacedertTrue(stat.isFile());
replacedertEquals(FILE_LEN, stat.getLen());
replacedertEquals(fc.makeQualified(FILE1), stat.getPath());
replacedertEquals(1, stat.getBlockLocations().length);
}
13
Source : TestINodeFile.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void checkEquals(RemoteIterator<LocatedFileStatus> i1, RemoteIterator<LocatedFileStatus> i2) throws IOException {
while (i1.hasNext()) {
replacedertTrue(i2.hasNext());
// Compare all the fields but the path name, which is relative
// to the original path from listFiles.
LocatedFileStatus l1 = i1.next();
LocatedFileStatus l2 = i2.next();
replacedertEquals(l1.getAccessTime(), l2.getAccessTime());
replacedertEquals(l1.getBlockSize(), l2.getBlockSize());
replacedertEquals(l1.getGroup(), l2.getGroup());
replacedertEquals(l1.getLen(), l2.getLen());
replacedertEquals(l1.getModificationTime(), l2.getModificationTime());
replacedertEquals(l1.getOwner(), l2.getOwner());
replacedertEquals(l1.getPermission(), l2.getPermission());
replacedertEquals(l1.getReplication(), l2.getReplication());
}
replacedertFalse(i2.hasNext());
}
13
Source : S3EventNotificationsPolicy.java
with Apache License 2.0
from mmolimar
with Apache License 2.0
from mmolimar
@Override
public Iterator<FileMetadata> listFiles(FileSystem fs) {
return sqs.receiveMessage(request).getMessages().stream().flatMap(message -> parseMessage(message).stream()).filter(record -> record.eventName.matches(eventNameRegex)).filter(record -> fs.getWorkingDirectory().toString().startsWith(getUriPrefix() + record.bucketName)).map(record -> {
Path path = new Path(getUriPrefix() + record.bucketName + "/", record.objectKey);
Optional<FileMetadata> metadata = Optional.empty();
try {
RemoteIterator<LocatedFileStatus> it = fs.listFiles(path, false);
if (it.hasNext()) {
LocatedFileStatus status = it.next();
if (status.isFile())
metadata = Optional.of(toMetadata(status));
}
} catch (Exception ioe) {
log.warn("{} Cannot get file at path '{}': {}", this, path, ioe.getMessage());
}
if (deleteMessages) {
log.trace("{} Removing message with ID '{}'.", this, record.messageId);
sqs.deleteMessage(queueUrl, record.receiptHandle);
}
return metadata;
}).flatMap(metadataOpt -> metadataOpt.map(Stream::of).orElseGet(Stream::empty)).iterator();
}
13
Source : RollingSinkITCase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* This uses {@link org.apache.flink.streaming.connectors.fs.DateTimeBucketer} to
* produce rolling files. The clock of DateTimeBucketer is set to
* {@link ModifyableClock} to keep the time in lockstep with the processing of elements using
* latches.
*/
@Test
public void testDateTimeRollingStringWriter() throws Exception {
final int numElements = 20;
final String outPath = hdfsURI + "/rolling-out";
DateTimeBucketer.setClock(new ModifyableClock());
ModifyableClock.setCurrentTime(0);
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
DataStream<Tuple2<Integer, String>> source = env.addSource(new WaitingTestSourceFunction(numElements)).broadcast();
// the parallel flatMap is chained to the sink, so when it has seen 5 elements it can
// fire the latch
DataStream<String> mapped = source.flatMap(new RichFlatMapFunction<Tuple2<Integer, String>, String>() {
private static final long serialVersionUID = 1L;
int count = 0;
@Override
public void flatMap(Tuple2<Integer, String> value, Collector<String> out) throws Exception {
out.collect(value.f1);
count++;
if (count >= 5) {
if (getRuntimeContext().getIndexOfThisSubtask() == 0) {
latch1.trigger();
} else {
latch2.trigger();
}
count = 0;
}
}
});
RollingSink<String> sink = new RollingSink<String>(outPath).setBucketer(new DateTimeBucketer("ss")).setPartPrefix("part").setPendingPrefix("").setPendingSuffix("");
mapped.addSink(sink);
env.execute("RollingSink String Write Test");
RemoteIterator<LocatedFileStatus> files = dfs.listFiles(new Path(outPath), true);
// we should have 8 rolling files, 4 time intervals and parallelism of 2
int numFiles = 0;
while (files.hasNext()) {
LocatedFileStatus file = files.next();
numFiles++;
if (file.getPath().toString().contains("rolling-out/00")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 0; i < 5; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/05")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 5; i < 10; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/10")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 10; i < 15; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else if (file.getPath().toString().contains("rolling-out/15")) {
FSDataInputStream inStream = dfs.open(file.getPath());
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
for (int i = 15; i < 20; i++) {
String line = br.readLine();
replacedert.replacedertEquals("message #" + i, line);
}
inStream.close();
} else {
replacedert.fail("File " + file + " does not match any expected roll pattern.");
}
}
replacedert.replacedertEquals(8, numFiles);
}
13
Source : TestCompressDecompress.java
with Apache License 2.0
from Intel-bigdata
with Apache License 2.0
from Intel-bigdata
@Test
public void testListLocatedStatus() throws Exception {
// if (!loadedNative()) {
// return;
// }
waitTillSSMExitSafeMode();
// initDB();
SmartFileSystem smartDfs = new SmartFileSystem();
smartDfs.initialize(dfs.getUri(), ssm.getContext().getConf());
int arraySize = 1024 * 1024 * 8;
String fileName = "/ssm/compression/file4";
byte[] bytes = prepareFile(fileName, arraySize);
// For uncompressed file, SmartFileSystem and DistributedFileSystem behave exactly the same
RemoteIterator<LocatedFileStatus> iter1 = dfs.listLocatedStatus(new Path(fileName));
LocatedFileStatus stat1 = iter1.next();
RemoteIterator<LocatedFileStatus> iter2 = smartDfs.listLocatedStatus(new Path(fileName));
LocatedFileStatus stat2 = iter2.next();
replacedert.replacedertEquals(stat1.getPath(), stat2.getPath());
replacedert.replacedertEquals(stat1.getBlockSize(), stat2.getBlockSize());
replacedert.replacedertEquals(stat1.getLen(), stat2.getLen());
BlockLocation[] blockLocations1 = stat1.getBlockLocations();
BlockLocation[] blockLocations2 = stat2.getBlockLocations();
replacedert.replacedertEquals(blockLocations1.length, blockLocations2.length);
for (int i = 0; i < blockLocations1.length; i++) {
replacedert.replacedertEquals(blockLocations1[i].getLength(), blockLocations2[i].getLength());
replacedert.replacedertEquals(blockLocations1[i].getOffset(), blockLocations2[i].getOffset());
}
// Test compressed file
int bufSize = 1024 * 1024;
CmdletManager cmdletManager = ssm.getCmdletManager();
long cmdId = cmdletManager.submitCmdlet("compress -file " + fileName + " -bufSize " + bufSize + " -codec " + codec);
waitTillActionDone(cmdId);
RemoteIterator<LocatedFileStatus> iter3 = dfs.listLocatedStatus(new Path(fileName));
LocatedFileStatus stat3 = iter3.next();
BlockLocation[] blockLocations3 = stat3.getBlockLocations();
RemoteIterator<LocatedFileStatus> iter4 = smartDfs.listLocatedStatus(new Path(fileName));
LocatedFileStatus stat4 = iter4.next();
BlockLocation[] blockLocations4 = stat4.getBlockLocations();
replacedert.replacedertEquals(stat1.getPath(), stat4.getPath());
replacedert.replacedertEquals(stat1.getBlockSize(), stat4.getBlockSize());
replacedert.replacedertEquals(stat1.getLen(), stat4.getLen());
}
13
Source : IntegrationTestLoadAndVerify.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
static SortedSet<byte[]> readKeysToSearch(final Configuration conf) throws IOException, InterruptedException {
Path keysInputDir = new Path(conf.get(SEARCHER_INPUTDIR_KEY));
FileSystem fs = FileSystem.get(conf);
SortedSet<byte[]> result = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
if (!fs.exists(keysInputDir)) {
throw new FileNotFoundException(keysInputDir.toString());
}
if (!fs.isDirectory(keysInputDir)) {
FileStatus keyFileStatus = fs.getFileStatus(keysInputDir);
readFileToSearch(conf, fs, keyFileStatus, result);
} else {
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(keysInputDir, false);
while (iterator.hasNext()) {
LocatedFileStatus keyFileStatus = iterator.next();
// Skip "_SUCCESS" file.
if (keyFileStatus.getPath().getName().startsWith("_"))
continue;
readFileToSearch(conf, fs, keyFileStatus, result);
}
}
return result;
}
13
Source : FileComponentsSource.java
with GNU General Public License v3.0
from dlanza1
with GNU General Public License v3.0
from dlanza1
public Properties loadProperties() throws Exception {
Properties props = new Properties();
RemoteIterator<LocatedFileStatus> fileStatusLisreplacederator = fs.listFiles(new Path(path), true);
while (fileStatusLisreplacederator.hasNext()) {
LocatedFileStatus fileStatus = fileStatusLisreplacederator.next();
if (fileStatus.isFile()) {
String filePath = fileStatus.getPath().toString();
LOG.info("Reading " + filePath);
try {
Properties fileProps = Properties.fromFile(filePath);
props.putAll(fileProps);
LOG.info("Loaded " + filePath);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
}
return props;
}
See More Examples