Here are the examples of the java api org.apache.hadoop.fs.FSDataInputStream taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
541 Examples
19
Source : HdfsIOBenchmark.java
with Apache License 2.0
from zrlio
with Apache License 2.0
from zrlio
private int read(FSDataInputStream stream, byte[] buf) throws IOException {
int off = 0;
int len = buf.length;
int ret = stream.read(buf, off, len);
while (ret > 0 && len - ret > 0) {
len -= ret;
off += ret;
ret = stream.read(buf, off, len);
}
return off > 0 || ret > 0 ? ret : -1;
}
19
Source : FooterGatherer.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
private static final void readFully(FSDataInputStream stream, long start, byte[] output, int offset, int len) throws IOException {
int bytesRead = 0;
while (bytesRead > -1 && bytesRead < len) {
bytesRead += stream.read(start + bytesRead, output, offset + bytesRead, len - bytesRead);
}
}
19
Source : ColumnDataReader.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
public clreplaced ColumnDataReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.clreplaced);
private final long endPosition;
public final FSDataInputStream input;
public ColumnDataReader(FSDataInputStream input, long start, long length) throws IOException {
this.input = input;
this.input.seek(start);
this.endPosition = start + length;
}
public PageHeader readPageHeader() throws IOException {
return Util.readPageHeader(input);
}
public FSDataInputStream getInputStream() {
return input;
}
public BytesInput getPageAsBytesInput(int pageLength) throws IOException {
byte[] b = new byte[pageLength];
input.read(b);
return new HadoopBytesInput(b);
}
public void loadPage(DrillBuf target, int pageLength) throws IOException {
target.clear();
ByteBuffer directBuffer = target.nioBuffer(0, pageLength);
int lengthLeftToRead = pageLength;
while (lengthLeftToRead > 0) {
lengthLeftToRead -= CompatibilityUtil.getBuf(input, directBuffer, lengthLeftToRead);
}
target.writerIndex(pageLength);
}
public void clear() {
try {
input.close();
} catch (IOException ex) {
logger.warn("Error while closing input stream.", ex);
}
}
public boolean hasRemainder() throws IOException {
return input.getPos() < endPosition;
}
public clreplaced HadoopBytesInput extends BytesInput {
private final byte[] pageBytes;
public HadoopBytesInput(byte[] pageBytes) {
super();
this.pageBytes = pageBytes;
}
@Override
public byte[] toByteArray() throws IOException {
return pageBytes;
}
@Override
public long size() {
return pageBytes.length;
}
@Override
public void writeAllTo(OutputStream out) throws IOException {
out.write(pageBytes);
}
}
}
19
Source : DrillFSDataInputStream.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
/**
* Wrapper around FSDataInputStream to collect IO Stats.
*/
public clreplaced DrillFSDataInputStream extends FSDataInputStream {
private final FSDataInputStream underlyingIs;
private final OpenFileTracker openFileTracker;
private final OperatorStats operatorStats;
public DrillFSDataInputStream(FSDataInputStream in, OperatorStats operatorStats) throws IOException {
this(in, operatorStats, null);
}
public DrillFSDataInputStream(FSDataInputStream in, OperatorStats operatorStats, OpenFileTracker openFileTracker) throws IOException {
super(new WrappedInputStream(in, operatorStats));
underlyingIs = in;
this.openFileTracker = openFileTracker;
this.operatorStats = operatorStats;
}
@Override
public synchronized void seek(long desired) throws IOException {
underlyingIs.seek(desired);
}
@Override
public long getPos() throws IOException {
return underlyingIs.getPos();
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
operatorStats.startWait();
try {
return underlyingIs.read(position, buffer, offset, length);
} finally {
operatorStats.stopWait();
}
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
operatorStats.startWait();
try {
underlyingIs.readFully(position, buffer, offset, length);
} finally {
operatorStats.stopWait();
}
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
operatorStats.startWait();
try {
underlyingIs.readFully(position, buffer);
} finally {
operatorStats.stopWait();
}
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return underlyingIs.seekToNewSource(targetPos);
}
@Override
@LimitedPrivate({ "HDFS" })
public InputStream getWrappedStream() {
return underlyingIs.getWrappedStream();
}
@Override
public int read(ByteBuffer buf) throws IOException {
operatorStats.startWait();
try {
return underlyingIs.read(buf);
} finally {
operatorStats.stopWait();
}
}
@Override
public FileDescriptor getFileDescriptor() throws IOException {
return underlyingIs.getFileDescriptor();
}
@Override
public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
underlyingIs.setReadahead(readahead);
}
@Override
public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
underlyingIs.setDropBehind(dropBehind);
}
@Override
public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
operatorStats.startWait();
try {
return underlyingIs.read(bufferPool, maxLength, opts);
} finally {
operatorStats.stopWait();
}
}
@Override
public void releaseBuffer(ByteBuffer buffer) {
underlyingIs.releaseBuffer(buffer);
}
@Override
public int read() throws IOException {
return underlyingIs.read();
}
@Override
public long skip(long n) throws IOException {
return underlyingIs.skip(n);
}
@Override
public int available() throws IOException {
return underlyingIs.available();
}
@Override
public void close() throws IOException {
if (openFileTracker != null) {
openFileTracker.fileClosed(this);
}
underlyingIs.close();
}
@Override
public void mark(int readlimit) {
underlyingIs.mark(readlimit);
}
@Override
public void reset() throws IOException {
underlyingIs.reset();
}
@Override
public boolean markSupported() {
return underlyingIs.markSupported();
}
@Override
public void unbuffer() {
underlyingIs.unbuffer();
}
/**
* We need to wrap the FSDataInputStream inside a InputStream, because read() method in InputStream is
* overridden in FilterInputStream (super clreplaced of FSDataInputStream) as final, so we can not override in
* DrillFSDataInputStream.
*/
private static clreplaced WrappedInputStream extends InputStream implements Seekable, PositionedReadable {
final FSDataInputStream is;
final OperatorStats operatorStats;
WrappedInputStream(FSDataInputStream is, OperatorStats operatorStats) {
this.is = is;
this.operatorStats = operatorStats;
}
/**
* Most of the read are going to be block reads which use {@link #read(byte[], int,
* int)}. So not adding stats for single byte reads.
*/
@Override
public int read() throws IOException {
return is.read();
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
operatorStats.startWait();
try {
return is.read(b, off, len);
} finally {
operatorStats.stopWait();
}
}
@Override
public int read(byte[] b) throws IOException {
operatorStats.startWait();
try {
return is.read(b);
} finally {
operatorStats.stopWait();
}
}
@Override
public long skip(long n) throws IOException {
return is.skip(n);
}
@Override
public int available() throws IOException {
return is.available();
}
@Override
public void close() throws IOException {
is.close();
}
@Override
public synchronized void mark(int readlimit) {
is.mark(readlimit);
}
@Override
public synchronized void reset() throws IOException {
is.reset();
}
@Override
public boolean markSupported() {
return is.markSupported();
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
return is.read(position, buffer, offset, length);
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
is.readFully(position, buffer, offset, length);
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
is.readFully(position, buffer);
}
@Override
public void seek(long pos) throws IOException {
is.seek(pos);
}
@Override
public long getPos() throws IOException {
return is.getPos();
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return is.seekToNewSource(targetPos);
}
}
}
19
Source : DFSUtil.java
with Apache License 2.0
from wgzhao
with Apache License 2.0
from wgzhao
// 判断file是否是Sequence file
private boolean isSequenceFile(Path filepath, FSDataInputStream in) {
final byte[] seqMagic = { (byte) 'S', (byte) 'E', (byte) 'Q' };
byte[] magic = new byte[seqMagic.length];
try {
in.seek(0);
in.readFully(magic);
return Arrays.equals(magic, seqMagic);
} catch (IOException e) {
LOG.info("检查文件类型: [{}] 不是Sequence File.", filepath);
}
return false;
}
19
Source : MetadataReader.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
private static InputStream readFully(FSDataInputStream from, long position, int length) throws IOException {
byte[] buffer = new byte[length];
from.readFully(position, buffer);
return new ByteArrayInputStream(buffer);
}
19
Source : HdfsOrcDataSource.java
with Apache License 2.0
from openlookeng
with Apache License 2.0
from openlookeng
public clreplaced HdfsOrcDataSource extends AbstractOrcDataSource {
private final FSDataInputStream inputStream;
private final FileFormatDataSourceStats stats;
public HdfsOrcDataSource(OrcDataSourceId id, long size, DataSize maxMergeDistance, DataSize maxReadSize, DataSize streamBufferSize, boolean lazyReadSmallRanges, FSDataInputStream inputStream, FileFormatDataSourceStats stats) {
super(id, size, maxMergeDistance, maxReadSize, streamBufferSize, lazyReadSmallRanges);
this.inputStream = requireNonNull(inputStream, "inputStream is null");
this.stats = requireNonNull(stats, "stats is null");
}
@Override
public void close() throws IOException {
inputStream.close();
}
@Override
protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength) {
try {
long readStart = System.nanoTime();
inputStream.readFully(position, buffer, bufferOffset, bufferLength);
stats.readDataBytesPerSecond(bufferLength, System.nanoTime() - readStart);
} catch (PrestoException e) {
// just in case there is a Presto wrapper or hook
throw e;
} catch (Exception e) {
String message = format("Error reading from %s at position %s", this, position);
if (e instanceof BlockMissingException) {
throw new PrestoException(HiveErrorCode.HIVE_MISSING_DATA, message, e);
}
if (e instanceof IOException) {
throw new PrestoException(HiveErrorCode.HIVE_FILESYSTEM_ERROR, message, e);
}
throw new PrestoException(HiveErrorCode.HIVE_UNKNOWN_ERROR, message, e);
}
}
}
19
Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static byte[] readAllWithRead4(FSDataInputStream fsdis, boolean close) throws IOException {
try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final byte[] buffer = new byte[17];
int totalRead = 0;
int read;
while (true) {
read = fsdis.read(totalRead, buffer, 0, buffer.length);
if (read > 0) {
totalRead += read;
baos.write(buffer, 0, read);
} else if (read < 0) {
// EOF
break;
} else {
// read == 0:
// zero result may be returned *only* in case if the 4th
// parameter is 0. Since in our case this is 'buffer.length',
// zero return value clearly indicates a bug:
throw new replacedertionError("FSDataInputStream#read(4) returned 0, while " + " the 4th method parameter is " + buffer.length + ".");
}
}
final byte[] result = baos.toByteArray();
return result;
} finally {
if (close) {
fsdis.close();
}
}
}
19
Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void expectSeekIOE(FSDataInputStream fsdis, long seekPos, String message) {
try {
fsdis.seek(seekPos);
replacedertTrue(message + " (Position = " + fsdis.getPos() + ")", false);
} catch (IOException ioe) {
// okay
}
}
19
Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static byte[] readAllWithReadFully(int totalLength, FSDataInputStream fsdis, boolean close) throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
// Simulate reading of some data structures of known length:
final byte[] buffer = new byte[17];
final int times = totalLength / buffer.length;
final int remainder = totalLength % buffer.length;
// it would be simpler to leave the position tracking to the
// InputStream, but we need to check the methods #readFully(2)
// and #readFully(4) that receive the position as a parameter:
int position = 0;
try {
// read "data structures":
for (int i = 0; i < times; i++) {
fsdis.readFully(position, buffer);
position += buffer.length;
baos.write(buffer);
}
if (remainder > 0) {
// read the remainder:
fsdis.readFully(position, buffer, 0, remainder);
position += remainder;
baos.write(buffer, 0, remainder);
}
try {
fsdis.readFully(position, buffer, 0, 1);
replacedertTrue(false);
} catch (IOException ioe) {
// okay
}
replacedertEquals(totalLength, position);
final byte[] result = baos.toByteArray();
replacedertEquals(totalLength, result.length);
return result;
} finally {
if (close) {
fsdis.close();
}
}
}
19
Source : TestRpcProgramNfs3.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private byte[] getFileContentsUsingDfs(String fileName, int len) throws Exception {
final FSDataInputStream in = hdfs.open(new Path(fileName));
final byte[] ret = new byte[len];
in.readFully(ret);
try {
in.readByte();
replacedert.fail("expected end of file");
} catch (EOFException e) {
// expected. Unfortunately there is no replacedociated message to check
}
in.close();
return ret;
}
19
Source : DFSClientCache.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
FSDataInputStream getDfsInputStream(String userName, String inodePath) {
DFSInputStreamCaheKey k = new DFSInputStreamCaheKey(userName, inodePath);
FSDataInputStream s = null;
try {
s = inputstreamCache.get(k);
} catch (ExecutionException e) {
LOG.warn("Failed to create DFSInputStream for user:" + userName + " Cause:" + e);
}
return s;
}
19
Source : TestPread.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void doPread(FSDataInputStream stm, long position, byte[] buffer, int offset, int length) throws IOException {
int nread = 0;
long totalRead = 0;
DFSInputStream dfstm = null;
if (stm.getWrappedStream() instanceof DFSInputStream) {
dfstm = (DFSInputStream) (stm.getWrappedStream());
totalRead = dfstm.getReadStatistics().getTotalBytesRead();
}
while (nread < length) {
int nbytes = stm.read(position + nread, buffer, offset + nread, length - nread);
replacedertTrue("Error in pread", nbytes > 0);
nread += nbytes;
}
if (dfstm != null) {
if (isHedgedRead) {
replacedertTrue("Expected read statistic to be incremented", length <= dfstm.getReadStatistics().getTotalBytesRead() - totalRead);
} else {
replacedertEquals("Expected read statistic to be incremented", length, dfstm.getReadStatistics().getTotalBytesRead() - totalRead);
}
}
}
19
Source : TestFileCreation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private byte[] readAll(FSDataInputStream in) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int n = 0;
while ((n = in.read(buffer)) > -1) {
out.write(buffer, 0, n);
}
return out.toByteArray();
}
19
Source : TestBlockTokenWithDFS.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
// read a file using blockSeekTo()
private boolean checkFile1(FSDataInputStream in) {
byte[] toRead = new byte[FILE_SIZE];
int totalRead = 0;
int nRead = 0;
try {
while ((nRead = in.read(toRead, totalRead, toRead.length - totalRead)) > 0) {
totalRead += nRead;
}
} catch (IOException e) {
return false;
}
replacedertEquals("Cannot read file.", toRead.length, totalRead);
return checkFile(toRead);
}
19
Source : AbstractContractOpenTest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Test Seek operations
*/
public abstract clreplaced AbstractContractOpenTest extends AbstractFSContractTestBase {
private FSDataInputStream instream;
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
conf.setInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, 4096);
return conf;
}
@Override
public void teardown() throws Exception {
IOUtils.closeStream(instream);
instream = null;
super.teardown();
}
@Test
public void testOpenReadZeroByteFile() throws Throwable {
describe("create & read a 0 byte file");
Path path = path("zero.txt");
touch(getFileSystem(), path);
instream = getFileSystem().open(path);
replacedertEquals(0, instream.getPos());
// expect initial read to fail
int result = instream.read();
replacedertMinusOne("initial byte read", result);
}
@Test
public void testFsIsEncrypted() throws Exception {
describe("create an empty file and call FileStatus.isEncrypted()");
final Path path = path("file");
createFile(getFileSystem(), path, false, new byte[0]);
final FileStatus stat = getFileSystem().getFileStatus(path);
replacedertFalse("Expecting false for stat.isEncrypted()", stat.isEncrypted());
}
@Test
public void testOpenReadDir() throws Throwable {
describe("create & read a directory");
Path path = path("zero.dir");
mkdirs(path);
try {
instream = getFileSystem().open(path);
// at this point we've opened a directory
fail("A directory has been opened for reading");
} catch (FileNotFoundException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("opening a directory for reading", "FileNotFoundException", e);
}
}
@Test
public void testOpenReadDirWithChild() throws Throwable {
describe("create & read a directory which has a child");
Path path = path("zero.dir");
mkdirs(path);
Path path2 = new Path(path, "child");
mkdirs(path2);
try {
instream = getFileSystem().open(path);
// at this point we've opened a directory
fail("A directory has been opened for reading");
} catch (FileNotFoundException e) {
handleExpectedException(e);
} catch (IOException e) {
handleRelaxedException("opening a directory for reading", "FileNotFoundException", e);
}
}
@Test
public void testOpenFileTwice() throws Throwable {
describe("verify that two opened file streams are independent");
Path path = path("testopenfiletwice.txt");
byte[] block = dataset(TEST_FILE_LEN, 0, 255);
// this file now has a simple rule: offset => value
createFile(getFileSystem(), path, false, block);
// open first
FSDataInputStream instream1 = getFileSystem().open(path);
int c = instream1.read();
replacedertEquals(0, c);
FSDataInputStream instream2 = null;
try {
instream2 = getFileSystem().open(path);
replacedertEquals("first read of instream 2", 0, instream2.read());
replacedertEquals("second read of instream 1", 1, instream1.read());
instream1.close();
replacedertEquals("second read of instream 2", 1, instream2.read());
// close instream1 again
instream1.close();
} finally {
IOUtils.closeStream(instream1);
IOUtils.closeStream(instream2);
}
}
@Test
public void testSequentialRead() throws Throwable {
describe("verify that sequential read() operations return values");
Path path = path("testsequentialread.txt");
int len = 4;
// 64
int base = 0x40;
byte[] block = dataset(len, base, base + len);
// this file now has a simple rule: offset => (value | 0x40)
createFile(getFileSystem(), path, false, block);
// open first
instream = getFileSystem().open(path);
replacedertEquals(base, instream.read());
replacedertEquals(base + 1, instream.read());
replacedertEquals(base + 2, instream.read());
replacedertEquals(base + 3, instream.read());
// and now, failures
replacedertEquals(-1, instream.read());
replacedertEquals(-1, instream.read());
instream.close();
}
}
19
Source : BoundedRangeFileInputStream.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop
* FSDataInputStream as a regular input stream. One can create multiple
* BoundedRangeFileInputStream on top of the same FSDataInputStream and they
* would not interfere with each other.
*/
clreplaced BoundedRangeFileInputStream extends InputStream {
private FSDataInputStream in;
private long pos;
private long end;
private long mark;
private final byte[] oneByte = new byte[1];
/**
* Constructor
*
* @param in
* The FSDataInputStream we connect to.
* @param offset
* Begining offset of the region.
* @param length
* Length of the region.
*
* The actual length of the region may be smaller if (off_begin +
* length) goes beyond the end of FS input stream.
*/
public BoundedRangeFileInputStream(FSDataInputStream in, long offset, long length) {
if (offset < 0 || length < 0) {
throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
}
this.in = in;
this.pos = offset;
this.end = offset + length;
this.mark = -1;
}
@Override
public int available() throws IOException {
int avail = in.available();
if (pos + avail > end) {
avail = (int) (end - pos);
}
return avail;
}
@Override
public int read() throws IOException {
int ret = read(oneByte);
if (ret == 1)
return oneByte[0] & 0xff;
return -1;
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
throw new IndexOutOfBoundsException();
}
int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
if (n == 0)
return -1;
int ret = 0;
synchronized (in) {
in.seek(pos);
ret = in.read(b, off, n);
}
if (ret < 0) {
end = pos;
return -1;
}
pos += ret;
return ret;
}
@Override
public /*
* We may skip beyond the end of the file.
*/
long skip(long n) throws IOException {
long len = Math.min(n, end - pos);
pos += len;
return len;
}
@Override
public synchronized void mark(int readlimit) {
mark = pos;
}
@Override
public synchronized void reset() throws IOException {
if (mark < 0)
throw new IOException("Resetting to invalid mark");
pos = mark;
}
@Override
public boolean markSupported() {
return true;
}
@Override
public void close() {
// Invalidate the state of the stream.
in = null;
pos = end;
mark = -1;
}
}
19
Source : TestReadAndSeekPageBlobAfterWrite.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* Read "size" bytes of data and verify that what was read and what was written
* are the same.
*/
private void readRandomDataAndVerify(int size) throws AzureException, IOException {
byte[] b = new byte[size];
FSDataInputStream stream = fs.open(PATH);
int bytesRead = stream.read(b);
stream.close();
replacedertEquals(bytesRead, size);
// compare the data read to the data written
replacedertTrue(comparePrefix(randomData, b, size));
}
19
Source : HadoopDataInputStreamTest.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Tests for the {@link HadoopDataInputStream}.
*/
public clreplaced HadoopDataInputStreamTest {
private FSDataInputStream verifyInputStream;
private HadoopDataInputStream testInputStream;
@Test
public void testSeekSkip() throws IOException {
verifyInputStream = spy(new FSDataInputStream(new SeekableByteArrayInputStream(new byte[2 * HadoopDataInputStream.MIN_SKIP_BYTES])));
testInputStream = new HadoopDataInputStream(verifyInputStream);
seekAndreplacedert(10);
seekAndreplacedert(10 + HadoopDataInputStream.MIN_SKIP_BYTES + 1);
seekAndreplacedert(testInputStream.getPos() - 1);
seekAndreplacedert(testInputStream.getPos() + 1);
seekAndreplacedert(testInputStream.getPos() - HadoopDataInputStream.MIN_SKIP_BYTES);
seekAndreplacedert(testInputStream.getPos());
seekAndreplacedert(0);
seekAndreplacedert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES);
seekAndreplacedert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES - 1);
try {
seekAndreplacedert(-1);
replacedert.fail();
} catch (Exception ignore) {
}
try {
seekAndreplacedert(-HadoopDataInputStream.MIN_SKIP_BYTES - 1);
replacedert.fail();
} catch (Exception ignore) {
}
}
private void seekAndreplacedert(long seekPos) throws IOException {
replacedert.replacedertEquals(verifyInputStream.getPos(), testInputStream.getPos());
long delta = seekPos - testInputStream.getPos();
testInputStream.seek(seekPos);
if (delta > 0L && delta <= HadoopDataInputStream.MIN_SKIP_BYTES) {
verify(verifyInputStream, atLeastOnce()).skip(anyLong());
verify(verifyInputStream, never()).seek(anyLong());
} else if (delta != 0L) {
verify(verifyInputStream, atLeastOnce()).seek(seekPos);
verify(verifyInputStream, never()).skip(anyLong());
} else {
verify(verifyInputStream, never()).seek(anyLong());
verify(verifyInputStream, never()).skip(anyLong());
}
replacedert.replacedertEquals(seekPos, verifyInputStream.getPos());
reset(verifyInputStream);
}
private static final clreplaced SeekableByteArrayInputStream extends ByteArrayInputStreamWithPos implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buffer) {
super(buffer);
}
@Override
public void seek(long pos) throws IOException {
setPosition((int) pos);
}
@Override
public long getPos() throws IOException {
return getPosition();
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
throw new UnsupportedOperationException();
}
}
}
19
Source : ColumnDataReader.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
public clreplaced ColumnDataReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.clreplaced);
private final long endPosition;
public final FSDataInputStream input;
public ColumnDataReader(FSDataInputStream input, long start, long length) throws IOException {
this.input = input;
this.input.seek(start);
this.endPosition = start + length;
}
public PageHeader readPageHeader() throws IOException {
return Util.readPageHeader(input);
}
public FSDataInputStream getInputStream() {
return input;
}
public BytesInput getPageAsBytesInput(int pageLength) throws IOException {
byte[] b = new byte[pageLength];
input.read(b);
return new HadoopBytesInput(b);
}
public void loadPage(DrillBuf target, int pageLength) throws IOException {
target.clear();
HadoopStreams.wrap(input).read(target.nioBuffer(0, pageLength));
target.writerIndex(pageLength);
}
public void clear() {
try {
input.close();
} catch (IOException ex) {
logger.warn("Error while closing input stream.", ex);
}
}
public boolean hasRemainder() throws IOException {
return input.getPos() < endPosition;
}
public clreplaced HadoopBytesInput extends BytesInput {
private final byte[] pageBytes;
public HadoopBytesInput(byte[] pageBytes) {
super();
this.pageBytes = pageBytes;
}
@Override
public byte[] toByteArray() throws IOException {
return pageBytes;
}
@Override
public long size() {
return pageBytes.length;
}
@Override
public void writeAllTo(OutputStream out) throws IOException {
out.write(pageBytes);
}
}
}
19
Source : FSInputGeneralColumnDataReader.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
public clreplaced FSInputGeneralColumnDataReader implements ColumnDataReader {
private FSDataInputStream fsInputStream;
private int numOfVals;
public FSInputGeneralColumnDataReader(FSDataInputStream fsInputStream, int dataStartOffset, int dataLength) throws IOException {
this.fsInputStream = fsInputStream;
fsInputStream.seek(dataStartOffset + dataLength - 4L);
this.numOfVals = fsInputStream.readInt();
fsInputStream.seek(dataStartOffset);
}
@Override
public byte[] read(int index) {
throw new UnsupportedOperationException("not support to read row operation");
}
@Override
public Iterator<byte[]> iterator() {
return new Iterator<byte[]>() {
int readRowCount = 0;
@Override
public boolean hasNext() {
return readRowCount < numOfVals;
}
@Override
public byte[] next() {
try {
int size = fsInputStream.readInt();
byte[] result = new byte[size];
fsInputStream.readFully(result);
readRowCount++;
return result;
} catch (IOException e) {
throw new NoSuchElementException("error when read data");
}
}
@Override
public void remove() {
throw new UnsupportedOperationException("unSupport operation");
}
};
}
@Override
public void close() throws IOException {
fsInputStream.close();
}
}
19
Source : FSInputNoCompressedColumnReader.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
public clreplaced FSInputNoCompressedColumnReader implements ColumnDataReader {
private FSDataInputStream fsInputStream;
private byte[] readBuffer;
private int colDataStartOffset;
private int colValLength;
private int rowCount;
public FSInputNoCompressedColumnReader(FSDataInputStream fsInputStream, int colDataStartOffset, int colValLength, int rowCount) throws IOException {
this.fsInputStream = fsInputStream;
this.colDataStartOffset = colDataStartOffset;
fsInputStream.seek(colDataStartOffset);
this.colValLength = colValLength;
this.rowCount = rowCount;
this.readBuffer = new byte[colValLength];
}
public Iterator<byte[]> iterator() {
return new NoCompressedColumnDataItr();
}
@Override
public byte[] read(int rowNum) {
throw new UnsupportedOperationException("not support to read row operation");
}
@Override
public void close() throws IOException {
fsInputStream.close();
}
private clreplaced NoCompressedColumnDataItr implements Iterator<byte[]> {
private int readRowCount = 0;
@Override
public boolean hasNext() {
return readRowCount < rowCount;
}
@Override
public byte[] next() {
try {
fsInputStream.readFully(readBuffer);
} catch (IOException e) {
throw new RuntimeException("error when read data", e);
}
readRowCount++;
return readBuffer;
}
@Override
public void remove() {
throw new UnsupportedOperationException("not supported");
}
}
}
19
Source : ColumnarStoreMetricsDesc.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
public ColumnDataReader getMetricsReaderFromFSInput(FSDataInputStream inputStream, int columnDataStartOffset, int columnDataLength, int rowCount) throws IOException {
if (Compression.LZ4 == compression && fixLen != -1) {
return new FSInputLZ4CompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
}
if (fixLen != -1) {
return new FSInputNoCompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength / rowCount, rowCount);
}
return new FSInputGeneralColumnDataReader(inputStream, columnDataStartOffset, columnDataLength);
}
19
Source : ColumnarStoreDimDesc.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
public ColumnDataReader getDimReaderFromFSInput(FSDataInputStream inputStream, int columnDataStartOffset, int columnDataLength, int rowCount) throws IOException {
if (compression == Compression.LZ4) {
return new FSInputLZ4CompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
} else if (compression == Compression.RUN_LENGTH) {
return new FSInputRLECompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
}
return new FSInputNoCompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength / rowCount, rowCount);
}
19
Source : HdfsDB.java
with MIT License
from fengdis
with MIT License
from fengdis
/**
* 下载文件
* @param path
* @param local
* @throws Exception
*/
public void downLoad(String path, String local) throws Exception {
FSDataInputStream in = fs.open(new Path(path));
OutputStream out = new FileOutputStream(local);
IOUtils.copyBytes(in, out, 4096, true);
}
19
Source : TestFileLink.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
private static void skipBuffer(FSDataInputStream in, byte v) throws IOException {
byte[] data = new byte[8192];
try {
int n;
while ((n = in.read(data)) == data.length) {
for (int i = 0; i < data.length; ++i) {
if (data[i] != v)
throw new Exception("File changed");
}
}
} catch (Exception e) {
}
}
19
Source : TestHFileBlockIndex.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
private void readIndex(boolean useTags) throws IOException {
long fileSize = fs.getFileStatus(path).getLen();
LOG.info("Size of " + path + ": " + fileSize);
FSDataInputStream istream = fs.open(path);
HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(true).withIncludesMvcc(includesMemstoreTS).withIncludesTags(useTags).withCompression(compr).build();
HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(istream, fs.getFileStatus(path).getLen(), meta);
BlockReaderWrapper brw = new BlockReaderWrapper(blockReader);
HFileBlockIndex.BlockIndexReader indexReader = new HFileBlockIndex.BlockIndexReader(KeyValue.RAW_COMPARATOR, numLevels, brw);
indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset, fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries);
long prevOffset = -1;
int i = 0;
int expectedHitCount = 0;
int expectedMissCount = 0;
LOG.info("Total number of keys: " + keys.size());
for (byte[] key : keys) {
replacedertTrue(key != null);
replacedertTrue(indexReader != null);
HFileBlock b = indexReader.seekToDataBlock(new KeyValue.KeyOnlyKeyValue(key, 0, key.length), null, true, true, false, null);
if (KeyValue.COMPARATOR.compareFlatKey(key, firstKeyInFile) < 0) {
replacedertTrue(b == null);
++i;
continue;
}
String keyStr = "key #" + i + ", " + Bytes.toStringBinary(key);
replacedertTrue("seekToDataBlock failed for " + keyStr, b != null);
if (prevOffset == b.getOffset()) {
replacedertEquals(++expectedHitCount, brw.hitCount);
} else {
LOG.info("First key in a new block: " + keyStr + ", block offset: " + b.getOffset() + ")");
replacedertTrue(b.getOffset() > prevOffset);
replacedertEquals(++expectedMissCount, brw.missCount);
prevOffset = b.getOffset();
}
++i;
}
istream.close();
}
19
Source : SnapshotManifest.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
/*
* Read the SnapshotDataManifest file
*/
private SnapshotDataManifest readDataManifest() throws IOException {
FSDataInputStream in = null;
try {
in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
CodedInputStream cin = CodedInputStream.newInstance(in);
cin.setSizeLimit(manifestSizeLimit);
return SnapshotDataManifest.parseFrom(cin);
} catch (FileNotFoundException e) {
return null;
} finally {
if (in != null)
in.close();
}
}
19
Source : BoundedRangeIndexFileInputStream.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
/**
* BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop
* FSDataInputStream as a regular input stream. One can create multiple
* BoundedRangeFileInputStream on top of the same FSDataInputStream and they
* would not interfere with each other.
* Copied from hadoop-335 tfile.
*/
clreplaced BoundedRangeIndexFileInputStream extends InputStream {
private FSDataInputStream in;
private long pos;
private long end;
private long mark;
private final byte[] oneByte = new byte[1];
private final boolean pread;
/**
* Constructor
*
* @param in
* The FSDataInputStream we connect to.
* @param offset
* Beginning offset of the region.
* @param length
* Length of the region.
* @param pread If true, use Filesystem positional read rather than seek+read.
*
* The actual length of the region may be smaller if (off_begin +
* length) goes beyond the end of FS input stream.
*/
public BoundedRangeIndexFileInputStream(FSDataInputStream in, long offset, long length, final boolean pread) {
if (offset < 0 || length < 0) {
throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
}
this.in = in;
this.pos = offset;
this.end = offset + length;
this.mark = -1;
this.pread = pread;
}
@Override
public int available() throws IOException {
int avail = in.available();
if (pos + avail > end) {
avail = (int) (end - pos);
}
return avail;
}
@Override
public int read() throws IOException {
int ret = read(oneByte);
if (ret == 1)
return oneByte[0] & 0xff;
return -1;
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
throw new IndexOutOfBoundsException();
}
int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
if (n == 0)
return -1;
int ret = 0;
if (this.pread) {
ret = in.read(pos, b, off, n);
} else {
synchronized (in) {
in.seek(pos);
ret = in.read(b, off, n);
}
}
if (ret < 0) {
end = pos;
return -1;
}
pos += ret;
return ret;
}
@Override
public /*
* We may skip beyond the end of the file.
*/
long skip(long n) throws IOException {
long len = Math.min(n, end - pos);
pos += len;
return len;
}
@Override
public void mark(int readlimit) {
mark = pos;
}
@Override
public void reset() throws IOException {
if (mark < 0)
throw new IOException("Resetting to invalid mark");
pos = mark;
}
@Override
public boolean markSupported() {
return true;
}
@Override
public void close() {
// Invalidate the state of the stream.
in = null;
pos = end;
mark = -1;
}
}
19
Source : HdfsSeekableInputStream.java
with MIT License
from epam
with MIT License
from epam
/**
* Source: HdfsSeekableInputStream.java
* Created: 4/29/2016
* Project: CATGenome Browser
* Make: IntelliJ IDEA 14.1.1, JDK 1.8
* <p>
* A {@link SeekableStream} extension, designed to read data from HDFS
* </p>
*/
public clreplaced HdfsSeekableInputStream extends SeekableStream {
private FSDataInputStream stream;
/**
* @param stream a HDFS {@link InputStream}
* @throws IOException
*/
public HdfsSeekableInputStream(InputStream stream) throws IOException {
this.stream = new FSDataInputStream(stream);
}
@Override
public long length() {
return 0;
}
@Override
public long position() throws IOException {
return stream.getPos();
}
@Override
public void seek(long position) throws IOException {
stream.seek(position);
}
@Override
public int read() throws IOException {
return stream.read();
}
@Override
public int read(byte[] buffer, int offset, int length) throws IOException {
return stream.read(buffer, offset, length);
}
@Override
public void close() throws IOException {
stream.close();
}
@Override
public boolean eof() throws IOException {
return false;
}
@Override
public String getSource() {
return null;
}
}
19
Source : FSDataInputStreamWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
public static FSInputStream of(FSDataInputStream in) throws IOException {
if (in.getWrappedStream() instanceof ByteBufferReadable) {
return new FSDataInputStreamWrapper(in);
}
return new ByteArrayFSInputStream(in);
}
19
Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
// See DX-15492
private void openNonExistentFileInPath(org.apache.hadoop.fs.Path f) throws IOException {
org.apache.hadoop.fs.Path nonExistentFile = f.suffix(NON_EXISTENT_FILE_SUFFIX + NON_EXISTENT_FILE_COUNTER++);
try (FSDataInputStream is = underlyingFs.open(nonExistentFile)) {
} catch (FileNotFoundException fileNotFoundException) {
return;
} catch (AccessControlException accessControlException) {
String errMsg = "Open failed for file: " + f.toString() + " error: Permission denied (13)";
throw new AccessControlException(errMsg);
} catch (Exception e) {
// Re-throw exception
throw e;
}
}
19
Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
FSInputStream newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
try {
return (operatorStats != null) ? com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWithStatsWrapper.of(is, operatorStats, true, f.toString()) : com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWrapper.of(is);
} catch (FSError e) {
throw propagateFSError(e);
}
}
19
Source : HadoopFileSystemWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
FSDataInputStreamWrapper newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
try {
return (operatorStats != null) ? new FSDataInputStreamWithStatsWrapper(is, operatorStats) : new FSDataInputStreamWrapper(is);
} catch (FSError e) {
throw propagateFSError(e);
}
}
19
Source : FSDataInputStreamWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
/**
* Wrapper around FSDataInputStream to capture {@code FSError}.
*/
clreplaced FSDataInputStreamWrapper extends FSDataInputStream {
private FSDataInputStream underlyingIs;
public FSDataInputStreamWrapper(FSDataInputStream in) throws IOException {
super(new WrappedInputStream(in));
underlyingIs = in;
}
protected FSDataInputStreamWrapper(FSDataInputStream in, WrappedInputStream wrapped) throws IOException {
super(wrapped);
underlyingIs = in;
}
protected FSDataInputStream getUnderlyingStream() {
return underlyingIs;
}
@Override
public synchronized void seek(long desired) throws IOException {
try {
underlyingIs.seek(desired);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public long getPos() throws IOException {
try {
return underlyingIs.getPos();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
try {
return underlyingIs.read(position, buffer, offset, length);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
try {
underlyingIs.readFully(position, buffer, offset, length);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
try {
underlyingIs.readFully(position, buffer);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
try {
return underlyingIs.seekToNewSource(targetPos);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
@LimitedPrivate({ "HDFS" })
public InputStream getWrappedStream() {
return underlyingIs.getWrappedStream();
}
@Override
public int read(ByteBuffer buf) throws IOException {
try {
return underlyingIs.read(buf);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
public int read(long position, ByteBuffer buf) throws IOException {
throw new UnsupportedOperationException("positioned read with ByteBuffer not supported");
}
@Override
public FileDescriptor getFileDescriptor() throws IOException {
try {
return underlyingIs.getFileDescriptor();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
try {
underlyingIs.setReadahead(readahead);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
try {
underlyingIs.setDropBehind(dropBehind);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
try {
return underlyingIs.read(bufferPool, maxLength, opts);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void releaseBuffer(ByteBuffer buffer) {
underlyingIs.releaseBuffer(buffer);
}
@Override
public int read() throws IOException {
try {
return underlyingIs.read();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public long skip(long n) throws IOException {
try {
return underlyingIs.skip(n);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public int available() throws IOException {
try {
return underlyingIs.available();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void close() throws IOException {
if (underlyingIs != null) {
try {
underlyingIs.close();
underlyingIs = null;
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
}
@Override
public void mark(int readlimit) {
underlyingIs.mark(readlimit);
}
@Override
public void reset() throws IOException {
try {
underlyingIs.reset();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public boolean markSupported() {
return underlyingIs.markSupported();
}
@Override
public void unbuffer() {
underlyingIs.unbuffer();
}
/**
* We need to wrap the FSDataInputStream inside a InputStream, because read() method in InputStream is
* overridden in FilterInputStream (super clreplaced of FSDataInputStream) as final, so we can not override in
* FSDataInputStreamWrapper.
*/
static clreplaced WrappedInputStream extends InputStream implements Seekable, PositionedReadable {
private FSDataInputStream is;
protected WrappedInputStream(FSDataInputStream is) {
this.is = is;
}
/**
* Most of the read are going to be block reads which use {@link #read(byte[], int,
* int)}. So not adding stats for single byte reads.
*/
@Override
public int read() throws IOException {
try {
return is.read();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
try {
return is.read(b, off, len);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public int read(byte[] b) throws IOException {
try {
return is.read(b);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public long skip(long n) throws IOException {
try {
return is.skip(n);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public int available() throws IOException {
try {
return is.available();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void close() throws IOException {
if (is != null) {
try {
is.close();
is = null;
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
}
@Override
public synchronized void mark(int readlimit) {
is.mark(readlimit);
}
@Override
public synchronized void reset() throws IOException {
try {
is.reset();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public boolean markSupported() {
return is.markSupported();
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
try {
return is.read(position, buffer, offset, length);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
try {
is.readFully(position, buffer, offset, length);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
try {
is.readFully(position, buffer);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public void seek(long pos) throws IOException {
try {
is.seek(pos);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public long getPos() throws IOException {
try {
return is.getPos();
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
try {
return is.seekToNewSource(targetPos);
} catch (FSError e) {
throw HadoopFileSystemWrapper.propagateFSError(e);
}
}
}
}
19
Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
com.dremio.io.FSInputStream newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
try {
return (operatorStats != null) ? com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWithStatsWrapper.of(is, operatorStats, true, f.toString()) : com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWrapper.of(is);
} catch (FSError e) {
throw propagateFSError(e);
}
}
19
Source : HdfsTransactionLog.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
clreplaced FSDataFastInputStream extends FastInputStream {
private FSDataInputStream fis;
public FSDataFastInputStream(FSDataInputStream fis, long chPosition) {
// super(null, new byte[10],0,0); // a small buffer size for testing purposes
super(null);
this.fis = fis;
super.readFromStream = chPosition;
}
@Override
public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
return fis.read(readFromStream, target, offset, len);
}
public void seek(long position) throws IOException {
if (position <= readFromStream && position >= getBufferPos()) {
// seek within buffer
pos = (int) (position - getBufferPos());
} else {
// long currSize = ch.size(); // not needed - underlying read should handle (unless read never done)
// if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + " on file of size " + currSize + " file=" + ch);
readFromStream = position;
end = pos = 0;
}
replacedert position() == position;
}
/**
* where is the start of the buffer relative to the whole file
*/
public long getBufferPos() {
return readFromStream - end;
}
public int getBufferSize() {
return buf.length;
}
@Override
public void close() throws IOException {
fis.close();
}
@Override
public String toString() {
return "readFromStream=" + readFromStream + " pos=" + pos + " end=" + end + " bufferPos=" + getBufferPos() + " position=" + position();
}
}
19
Source : AbstractContractUnbufferTest.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
protected void validateFullFileContents(FSDataInputStream stream) throws IOException {
replacedertNotNull(stream);
validateFileContents(stream, TEST_FILE_LEN, 0);
}
19
Source : HadoopUtils.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* cat file on hdfs
*
* @param hdfsFilePath hdfs file path
* @return byte[] byte array
* @throws IOException errors
*/
public byte[] catFile(String hdfsFilePath) throws IOException {
if (StringUtils.isBlank(hdfsFilePath)) {
logger.error("hdfs file path:{} is blank", hdfsFilePath);
return new byte[0];
}
try (FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath))) {
return IOUtils.toByteArray(fsDataInputStream);
}
}
19
Source : HadoopStreams.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Wraps a {@link FSDataInputStream} in a {@link SeekableInputStream} implementation for readers.
*
* @param stream a Hadoop FSDataInputStream
* @return a SeekableInputStream
*/
static SeekableInputStream wrap(FSDataInputStream stream) {
return new HadoopSeekableInputStream(stream);
}
19
Source : InLineFsDataInputStream.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Inline {@link FSDataInputStream}. A startOffset that is preplaceded in is replacedumed to be the start of the InputStream.
* All operations are handled having the {@code startOffset} as starting point.
*/
public clreplaced InLineFsDataInputStream extends FSDataInputStream {
private final int startOffset;
private final FSDataInputStream outerStream;
private final int length;
public InLineFsDataInputStream(int startOffset, FSDataInputStream outerStream, int length) throws IOException {
super(outerStream.getWrappedStream());
this.startOffset = startOffset;
this.outerStream = outerStream;
this.length = length;
outerStream.seek(startOffset);
}
@Override
public void seek(long desired) throws IOException {
if (desired > length) {
throw new IOException("Attempting to seek past inline content");
}
outerStream.seek(startOffset + desired);
}
@Override
public long getPos() throws IOException {
return outerStream.getPos() - startOffset;
}
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
if ((length + offset) > this.length) {
throw new IOException("Attempting to read past inline content");
}
return outerStream.read(startOffset + position, buffer, offset, length);
}
@Override
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
if ((length + offset) > this.length) {
throw new IOException("Attempting to read past inline content");
}
outerStream.readFully(startOffset + position, buffer, offset, length);
}
@Override
public void readFully(long position, byte[] buffer) throws IOException {
readFully(position, buffer, 0, buffer.length);
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
if (targetPos > this.length) {
throw new IOException("Attempting to seek past inline content");
}
return outerStream.seekToNewSource(startOffset + targetPos);
}
@Override
public int read(ByteBuffer buf) throws IOException {
return outerStream.read(buf);
}
@Override
public FileDescriptor getFileDescriptor() throws IOException {
return outerStream.getFileDescriptor();
}
@Override
public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
if (readahead > this.length) {
throw new IOException("Attempting to set read ahead past inline content");
}
outerStream.setReadahead(readahead);
}
@Override
public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
outerStream.setDropBehind(dropBehind);
}
@Override
public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
if (maxLength > this.length) {
throw new IOException("Attempting to read max length beyond inline content");
}
return outerStream.read(bufferPool, maxLength, opts);
}
@Override
public void releaseBuffer(ByteBuffer buffer) {
outerStream.releaseBuffer(buffer);
}
@Override
public void unbuffer() {
outerStream.unbuffer();
}
}
19
Source : HoodieWrapperFileSystem.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private FSDataInputStream wrapInputStream(final Path path, FSDataInputStream fsDataInputStream) throws IOException {
if (fsDataInputStream instanceof TimedFSDataInputStream) {
return fsDataInputStream;
}
return new TimedFSDataInputStream(path, fsDataInputStream);
}
18
Source : ColumnChunkIncReadStore.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
public void close() throws IOException {
for (FSDataInputStream stream : streams) {
stream.close();
}
for (ColumnChunkIncPageReader reader : columns.values()) {
reader.close();
}
}
18
Source : TextInput.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
/**
* Clreplaced that fronts an InputStream to provide a byte consumption interface.
* Also manages only reading lines to and from each split.
*/
final clreplaced TextInput {
private final byte[] lineSeparator;
private final byte normalizedLineSeparator;
private final TextParsingSettings settings;
private long lineCount;
private long charCount;
/**
* The starting position in the file.
*/
private final long startPos;
private final long endPos;
private int bufferMark;
private long streamMark;
private long streamPos;
private final Seekable seekable;
private final FSDataInputStream inputFS;
private final InputStream input;
private final DrillBuf buffer;
private final ByteBuffer underlyingBuffer;
private final long bStart;
private final long bStartMinus1;
private final boolean bufferReadable;
/**
* Whether there was a possible partial line separator on the previous
* read so we dropped it and it should be appended to next read.
*/
private int remByte = -1;
/**
* The current position in the buffer.
*/
public int bufferPtr;
/**
* The quanreplacedy of valid data in the buffer.
*/
public int length = -1;
private boolean endFound = false;
/**
* Creates a new instance with the mandatory characters for handling newlines transparently.
* lineSeparator the sequence of characters that represent a newline, as defined in {@link Format#getLineSeparator()}
* normalizedLineSeparator the normalized newline character (as defined in {@link Format#getNormalizedNewline()}) that is used to replace any lineSeparator sequence found in the input.
*/
public TextInput(TextParsingSettings settings, InputStream input, DrillBuf readBuffer, long startPos, long endPos) {
this.lineSeparator = settings.getNewLineDelimiter();
byte normalizedLineSeparator = settings.getNormalizedNewLine();
Preconditions.checkArgument(input instanceof Seekable, "Text input only supports an InputStream that supports Seekable.");
boolean isCompressed = input instanceof CompressionInputStream;
Preconditions.checkArgument(!isCompressed || startPos == 0, "Cannot use split on compressed stream.");
// splits aren't allowed with compressed data. The split length will be the compressed size which means we'll normally end prematurely.
if (isCompressed && endPos > 0) {
endPos = Long.MAX_VALUE;
}
this.input = input;
this.seekable = (Seekable) input;
this.settings = settings;
if (input instanceof FSDataInputStream) {
this.inputFS = (FSDataInputStream) input;
this.bufferReadable = inputFS.getWrappedStream() instanceof ByteBufferReadable;
} else {
this.inputFS = null;
this.bufferReadable = false;
}
this.startPos = startPos;
this.endPos = endPos;
this.normalizedLineSeparator = normalizedLineSeparator;
this.buffer = readBuffer;
this.bStart = buffer.memoryAddress();
this.bStartMinus1 = bStart - 1;
this.underlyingBuffer = buffer.nioBuffer(0, buffer.capacity());
}
/**
* Test the input to position for read start. If the input is a non-zero split or
* splitFirstLine is enabled, input will move to appropriate complete line.
* @throws IOException
*/
final void start() throws IOException {
lineCount = 0;
if (startPos > 0) {
seekable.seek(startPos);
}
updateBuffer();
if (length > 0) {
if (startPos > 0 || settings.isSkipFirstLine()) {
// move to next full record.
skipLines(1);
}
}
}
/**
* Helper method to get the most recent characters consumed since the last record started.
* May get an incomplete string since we don't support stream rewind. Returns empty string for now.
* @return String of last few bytes.
* @throws IOException
*/
public String getStringSinceMarkForError() throws IOException {
return " ";
}
long getPos() {
return streamPos + bufferPtr;
}
public void mark() {
streamMark = streamPos;
bufferMark = bufferPtr;
}
/**
* read some more bytes from the stream. Uses the zero copy interface if available. Otherwise, does byte copy.
* @throws IOException
*/
private void read() throws IOException {
if (bufferReadable) {
if (remByte != -1) {
for (int i = 0; i <= remByte; i++) {
underlyingBuffer.put(lineSeparator[i]);
}
remByte = -1;
}
length = inputFS.read(underlyingBuffer);
} else {
byte[] b = new byte[underlyingBuffer.capacity()];
if (remByte != -1) {
int remBytesNum = remByte + 1;
System.arraycopy(lineSeparator, 0, b, 0, remBytesNum);
length = input.read(b, remBytesNum, b.length - remBytesNum);
remByte = -1;
} else {
length = input.read(b);
}
underlyingBuffer.put(b);
}
}
/**
* Read more data into the buffer. Will also manage split end conditions.
* @throws IOException
*/
private void updateBuffer() throws IOException {
streamPos = seekable.getPos();
underlyingBuffer.clear();
if (endFound) {
length = -1;
return;
}
read();
// check our data read allowance.
if (streamPos + length >= this.endPos) {
updateLengthBasedOnConstraint();
}
charCount += bufferPtr;
bufferPtr = 1;
buffer.writerIndex(underlyingBuffer.limit());
buffer.readerIndex(underlyingBuffer.position());
}
/**
* Checks to see if we can go over the end of our bytes constraint on the data. If so,
* adjusts so that we can only read to the last character of the first line that crosses
* the split boundary.
*/
private void updateLengthBasedOnConstraint() {
final long max = bStart + length;
for (long m = bStart + (endPos - streamPos); m < max; m++) {
for (int i = 0; i < lineSeparator.length; i++) {
long mPlus = m + i;
if (mPlus < max) {
// we found a line separator and don't need to consult the next byte.
if (lineSeparator[i] == PlatformDependent.getByte(mPlus) && i == lineSeparator.length - 1) {
length = (int) (mPlus - bStart) + 1;
endFound = true;
return;
}
} else {
// the last N characters of the read were remnant bytes. We'll hold off on dealing with these bytes until the next read.
remByte = i;
length = length - i;
return;
}
}
}
}
/**
* Get next byte from stream. Also maintains the current line count. Will throw a StreamFinishedPseudoException
* when the stream has run out of bytes.
* @return next byte from stream.
* @throws IOException
*/
public final byte nextChar() throws IOException {
byte byteChar = nextCharNoNewLineCheck();
int bufferPtrTemp = bufferPtr - 1;
if (byteChar == lineSeparator[0]) {
for (int i = 1; i < lineSeparator.length; i++, bufferPtrTemp++) {
if (lineSeparator[i] != buffer.getByte(bufferPtrTemp)) {
return byteChar;
}
}
lineCount++;
byteChar = normalizedLineSeparator;
// we don't need to update buffer position if line separator is one byte long
if (lineSeparator.length > 1) {
bufferPtr += (lineSeparator.length - 1);
if (bufferPtr >= length) {
if (length != -1) {
updateBuffer();
} else {
throw StreamFinishedPseudoException.INSTANCE;
}
}
}
}
return byteChar;
}
/**
* Get next byte from stream. Do no maintain any line count Will throw a StreamFinishedPseudoException
* when the stream has run out of bytes.
* @return next byte from stream.
* @throws IOException
*/
public final byte nextCharNoNewLineCheck() throws IOException {
if (length == -1) {
throw StreamFinishedPseudoException.INSTANCE;
}
if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
buffer.checkBytes(bufferPtr - 1, bufferPtr);
}
byte byteChar = PlatformDependent.getByte(bStartMinus1 + bufferPtr);
if (bufferPtr >= length) {
if (length != -1) {
updateBuffer();
bufferPtr--;
} else {
throw StreamFinishedPseudoException.INSTANCE;
}
}
bufferPtr++;
return byteChar;
}
/**
* Number of lines read since the start of this split.
* @return
*/
public final long lineCount() {
return lineCount;
}
/**
* Skip forward the number of line delimiters. If you are in the middle of a line,
* a value of 1 will skip to the start of the next record.
* @param lines Number of lines to skip.
* @throws IOException
*/
public final void skipLines(int lines) throws IOException {
if (lines < 1) {
return;
}
long expectedLineCount = this.lineCount + lines;
try {
do {
nextChar();
} while (lineCount < expectedLineCount);
if (lineCount < lines) {
throw new IllegalArgumentException("Unable to skip " + lines + " lines from line " + (expectedLineCount - lines) + ". End of input reached");
}
} catch (EOFException ex) {
throw new IllegalArgumentException("Unable to skip " + lines + " lines from line " + (expectedLineCount - lines) + ". End of input reached");
}
}
public final long charCount() {
return charCount + bufferPtr;
}
public long getLineCount() {
return lineCount;
}
public void close() throws IOException {
input.close();
}
}
18
Source : DFSUtil.java
with Apache License 2.0
from wgzhao
with Apache License 2.0
from wgzhao
// 判断file是否是RC file
private boolean isRCFile(String filepath, FSDataInputStream in) {
// The first version of RCFile used the sequence file header.
final byte[] originalMagic = { (byte) 'S', (byte) 'E', (byte) 'Q' };
// The 'magic' bytes at the beginning of the RCFile
final byte[] rcMagic = { (byte) 'R', (byte) 'C', (byte) 'F' };
// the version that was included with the original magic, which is mapped
// into ORIGINAL_VERSION
final byte ORIGINAL_MAGIC_VERSION_WITH_METADATA = 6;
// All of the versions should be place in this list.
// version with SEQ
final int ORIGINAL_VERSION = 0;
// version with RCF
// final int NEW_MAGIC_VERSION = 1
// final int CURRENT_VERSION = NEW_MAGIC_VERSION
final int CURRENT_VERSION = 1;
byte version;
byte[] magic = new byte[rcMagic.length];
try {
in.seek(0);
in.readFully(magic);
if (Arrays.equals(magic, originalMagic)) {
byte vers = in.readByte();
if (vers != ORIGINAL_MAGIC_VERSION_WITH_METADATA) {
return false;
}
version = ORIGINAL_VERSION;
} else {
if (!Arrays.equals(magic, rcMagic)) {
return false;
}
// Set 'version'
version = in.readByte();
if (version > CURRENT_VERSION) {
return false;
}
}
if (version == ORIGINAL_VERSION) {
try {
Clreplaced<?> keyCls = hadoopConf.getClreplacedByName(Text.readString(in));
Clreplaced<?> valCls = hadoopConf.getClreplacedByName(Text.readString(in));
if (!keyCls.equals(RCFile.KeyBuffer.clreplaced) || !valCls.equals(RCFile.ValueBuffer.clreplaced)) {
return false;
}
} catch (ClreplacedNotFoundException e) {
return false;
}
}
// boolean decompress = in.readBoolean(); // is compressed?
if (version == ORIGINAL_VERSION) {
// is block-compressed? it should be always false.
boolean blkCompressed = in.readBoolean();
return !blkCompressed;
}
return true;
} catch (IOException e) {
LOG.info("检查文件类型: [{}] 不是RC File.", filepath);
}
return false;
}
18
Source : DFSUtil.java
with Apache License 2.0
from wgzhao
with Apache License 2.0
from wgzhao
// 判断是否为parquet
private boolean isParquetFile(Path file, FSDataInputStream in) {
try {
GroupReadSupport readSupport = new GroupReadSupport();
ParquetReader.Builder<Group> reader = ParquetReader.builder(readSupport, file);
ParquetReader<Group> build = reader.build();
if (build.read() != null) {
return true;
}
} catch (IOException e) {
LOG.info("检查文件类型: [{}] 不是Parquet File.", file);
}
return false;
}
18
Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.clreplaced, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_FORBIDDEN + ".*")
public void testReadForbidden() throws Exception {
try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(HTTP_FORBIDDEN);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
inputStream.read();
}
}
}
18
Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.clreplaced, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_NOT_FOUND + ".*")
public void testReadNotFound() throws Exception {
try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(HTTP_NOT_FOUND);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
inputStream.read();
}
}
}
18
Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
@Test
public void testReadRequestRangeNotSatisfiable() throws Exception {
try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(HTTP_RANGE_NOT_SATISFIABLE);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
replacedertEquals(inputStream.read(), -1);
}
}
}
18
Source : FSDataInputStreamTail.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
public static long readTailForFileSize(String path, long paddedFileSize, FSDataInputStream inputStream) throws IOException {
long position = max(paddedFileSize - MAX_SUPPORTED_PADDING_BYTES, 0);
long maxEOFAt = paddedFileSize + 1;
long startPos = inputStream.getPos();
try {
inputStream.seek(position);
int c;
while (position < maxEOFAt) {
c = inputStream.read();
if (c < 0) {
return position;
}
position++;
}
throw rejectInvalidFileSize(path, paddedFileSize);
} finally {
inputStream.seek(startPos);
}
}
See More Examples