org.apache.hadoop.fs.FSDataInputStream

Here are the examples of the java api org.apache.hadoop.fs.FSDataInputStream taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

541 Examples 7

19 Source : HdfsIOBenchmark.java
with Apache License 2.0
from zrlio

private int read(FSDataInputStream stream, byte[] buf) throws IOException {
    int off = 0;
    int len = buf.length;
    int ret = stream.read(buf, off, len);
    while (ret > 0 && len - ret > 0) {
        len -= ret;
        off += ret;
        ret = stream.read(buf, off, len);
    }
    return off > 0 || ret > 0 ? ret : -1;
}

19 Source : FooterGatherer.java
with Apache License 2.0
from zpochen

private static final void readFully(FSDataInputStream stream, long start, byte[] output, int offset, int len) throws IOException {
    int bytesRead = 0;
    while (bytesRead > -1 && bytesRead < len) {
        bytesRead += stream.read(start + bytesRead, output, offset + bytesRead, len - bytesRead);
    }
}

19 Source : ColumnDataReader.java
with Apache License 2.0
from zpochen

public clreplaced ColumnDataReader {

    static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.clreplaced);

    private final long endPosition;

    public final FSDataInputStream input;

    public ColumnDataReader(FSDataInputStream input, long start, long length) throws IOException {
        this.input = input;
        this.input.seek(start);
        this.endPosition = start + length;
    }

    public PageHeader readPageHeader() throws IOException {
        return Util.readPageHeader(input);
    }

    public FSDataInputStream getInputStream() {
        return input;
    }

    public BytesInput getPageAsBytesInput(int pageLength) throws IOException {
        byte[] b = new byte[pageLength];
        input.read(b);
        return new HadoopBytesInput(b);
    }

    public void loadPage(DrillBuf target, int pageLength) throws IOException {
        target.clear();
        ByteBuffer directBuffer = target.nioBuffer(0, pageLength);
        int lengthLeftToRead = pageLength;
        while (lengthLeftToRead > 0) {
            lengthLeftToRead -= CompatibilityUtil.getBuf(input, directBuffer, lengthLeftToRead);
        }
        target.writerIndex(pageLength);
    }

    public void clear() {
        try {
            input.close();
        } catch (IOException ex) {
            logger.warn("Error while closing input stream.", ex);
        }
    }

    public boolean hasRemainder() throws IOException {
        return input.getPos() < endPosition;
    }

    public clreplaced HadoopBytesInput extends BytesInput {

        private final byte[] pageBytes;

        public HadoopBytesInput(byte[] pageBytes) {
            super();
            this.pageBytes = pageBytes;
        }

        @Override
        public byte[] toByteArray() throws IOException {
            return pageBytes;
        }

        @Override
        public long size() {
            return pageBytes.length;
        }

        @Override
        public void writeAllTo(OutputStream out) throws IOException {
            out.write(pageBytes);
        }
    }
}

19 Source : DrillFSDataInputStream.java
with Apache License 2.0
from zpochen

/**
 * Wrapper around FSDataInputStream to collect IO Stats.
 */
public clreplaced DrillFSDataInputStream extends FSDataInputStream {

    private final FSDataInputStream underlyingIs;

    private final OpenFileTracker openFileTracker;

    private final OperatorStats operatorStats;

    public DrillFSDataInputStream(FSDataInputStream in, OperatorStats operatorStats) throws IOException {
        this(in, operatorStats, null);
    }

    public DrillFSDataInputStream(FSDataInputStream in, OperatorStats operatorStats, OpenFileTracker openFileTracker) throws IOException {
        super(new WrappedInputStream(in, operatorStats));
        underlyingIs = in;
        this.openFileTracker = openFileTracker;
        this.operatorStats = operatorStats;
    }

    @Override
    public synchronized void seek(long desired) throws IOException {
        underlyingIs.seek(desired);
    }

    @Override
    public long getPos() throws IOException {
        return underlyingIs.getPos();
    }

    @Override
    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
        operatorStats.startWait();
        try {
            return underlyingIs.read(position, buffer, offset, length);
        } finally {
            operatorStats.stopWait();
        }
    }

    @Override
    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
        operatorStats.startWait();
        try {
            underlyingIs.readFully(position, buffer, offset, length);
        } finally {
            operatorStats.stopWait();
        }
    }

    @Override
    public void readFully(long position, byte[] buffer) throws IOException {
        operatorStats.startWait();
        try {
            underlyingIs.readFully(position, buffer);
        } finally {
            operatorStats.stopWait();
        }
    }

    @Override
    public boolean seekToNewSource(long targetPos) throws IOException {
        return underlyingIs.seekToNewSource(targetPos);
    }

    @Override
    @LimitedPrivate({ "HDFS" })
    public InputStream getWrappedStream() {
        return underlyingIs.getWrappedStream();
    }

    @Override
    public int read(ByteBuffer buf) throws IOException {
        operatorStats.startWait();
        try {
            return underlyingIs.read(buf);
        } finally {
            operatorStats.stopWait();
        }
    }

    @Override
    public FileDescriptor getFileDescriptor() throws IOException {
        return underlyingIs.getFileDescriptor();
    }

    @Override
    public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
        underlyingIs.setReadahead(readahead);
    }

    @Override
    public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
        underlyingIs.setDropBehind(dropBehind);
    }

    @Override
    public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
        operatorStats.startWait();
        try {
            return underlyingIs.read(bufferPool, maxLength, opts);
        } finally {
            operatorStats.stopWait();
        }
    }

    @Override
    public void releaseBuffer(ByteBuffer buffer) {
        underlyingIs.releaseBuffer(buffer);
    }

    @Override
    public int read() throws IOException {
        return underlyingIs.read();
    }

    @Override
    public long skip(long n) throws IOException {
        return underlyingIs.skip(n);
    }

    @Override
    public int available() throws IOException {
        return underlyingIs.available();
    }

    @Override
    public void close() throws IOException {
        if (openFileTracker != null) {
            openFileTracker.fileClosed(this);
        }
        underlyingIs.close();
    }

    @Override
    public void mark(int readlimit) {
        underlyingIs.mark(readlimit);
    }

    @Override
    public void reset() throws IOException {
        underlyingIs.reset();
    }

    @Override
    public boolean markSupported() {
        return underlyingIs.markSupported();
    }

    @Override
    public void unbuffer() {
        underlyingIs.unbuffer();
    }

    /**
     * We need to wrap the FSDataInputStream inside a InputStream, because read() method in InputStream is
     * overridden in FilterInputStream (super clreplaced of FSDataInputStream) as final, so we can not override in
     * DrillFSDataInputStream.
     */
    private static clreplaced WrappedInputStream extends InputStream implements Seekable, PositionedReadable {

        final FSDataInputStream is;

        final OperatorStats operatorStats;

        WrappedInputStream(FSDataInputStream is, OperatorStats operatorStats) {
            this.is = is;
            this.operatorStats = operatorStats;
        }

        /**
         * Most of the read are going to be block reads which use {@link #read(byte[], int,
         * int)}. So not adding stats for single byte reads.
         */
        @Override
        public int read() throws IOException {
            return is.read();
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException {
            operatorStats.startWait();
            try {
                return is.read(b, off, len);
            } finally {
                operatorStats.stopWait();
            }
        }

        @Override
        public int read(byte[] b) throws IOException {
            operatorStats.startWait();
            try {
                return is.read(b);
            } finally {
                operatorStats.stopWait();
            }
        }

        @Override
        public long skip(long n) throws IOException {
            return is.skip(n);
        }

        @Override
        public int available() throws IOException {
            return is.available();
        }

        @Override
        public void close() throws IOException {
            is.close();
        }

        @Override
        public synchronized void mark(int readlimit) {
            is.mark(readlimit);
        }

        @Override
        public synchronized void reset() throws IOException {
            is.reset();
        }

        @Override
        public boolean markSupported() {
            return is.markSupported();
        }

        @Override
        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
            return is.read(position, buffer, offset, length);
        }

        @Override
        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
            is.readFully(position, buffer, offset, length);
        }

        @Override
        public void readFully(long position, byte[] buffer) throws IOException {
            is.readFully(position, buffer);
        }

        @Override
        public void seek(long pos) throws IOException {
            is.seek(pos);
        }

        @Override
        public long getPos() throws IOException {
            return is.getPos();
        }

        @Override
        public boolean seekToNewSource(long targetPos) throws IOException {
            return is.seekToNewSource(targetPos);
        }
    }
}

19 Source : DFSUtil.java
with Apache License 2.0
from wgzhao

// 判断file是否是Sequence file
private boolean isSequenceFile(Path filepath, FSDataInputStream in) {
    final byte[] seqMagic = { (byte) 'S', (byte) 'E', (byte) 'Q' };
    byte[] magic = new byte[seqMagic.length];
    try {
        in.seek(0);
        in.readFully(magic);
        return Arrays.equals(magic, seqMagic);
    } catch (IOException e) {
        LOG.info("检查文件类型: [{}] 不是Sequence File.", filepath);
    }
    return false;
}

19 Source : MetadataReader.java
with Apache License 2.0
from openlookeng

private static InputStream readFully(FSDataInputStream from, long position, int length) throws IOException {
    byte[] buffer = new byte[length];
    from.readFully(position, buffer);
    return new ByteArrayInputStream(buffer);
}

19 Source : HdfsOrcDataSource.java
with Apache License 2.0
from openlookeng

public clreplaced HdfsOrcDataSource extends AbstractOrcDataSource {

    private final FSDataInputStream inputStream;

    private final FileFormatDataSourceStats stats;

    public HdfsOrcDataSource(OrcDataSourceId id, long size, DataSize maxMergeDistance, DataSize maxReadSize, DataSize streamBufferSize, boolean lazyReadSmallRanges, FSDataInputStream inputStream, FileFormatDataSourceStats stats) {
        super(id, size, maxMergeDistance, maxReadSize, streamBufferSize, lazyReadSmallRanges);
        this.inputStream = requireNonNull(inputStream, "inputStream is null");
        this.stats = requireNonNull(stats, "stats is null");
    }

    @Override
    public void close() throws IOException {
        inputStream.close();
    }

    @Override
    protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength) {
        try {
            long readStart = System.nanoTime();
            inputStream.readFully(position, buffer, bufferOffset, bufferLength);
            stats.readDataBytesPerSecond(bufferLength, System.nanoTime() - readStart);
        } catch (PrestoException e) {
            // just in case there is a Presto wrapper or hook
            throw e;
        } catch (Exception e) {
            String message = format("Error reading from %s at position %s", this, position);
            if (e instanceof BlockMissingException) {
                throw new PrestoException(HiveErrorCode.HIVE_MISSING_DATA, message, e);
            }
            if (e instanceof IOException) {
                throw new PrestoException(HiveErrorCode.HIVE_FILESYSTEM_ERROR, message, e);
            }
            throw new PrestoException(HiveErrorCode.HIVE_UNKNOWN_ERROR, message, e);
        }
    }
}

19 Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB

private static byte[] readAllWithRead4(FSDataInputStream fsdis, boolean close) throws IOException {
    try {
        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        final byte[] buffer = new byte[17];
        int totalRead = 0;
        int read;
        while (true) {
            read = fsdis.read(totalRead, buffer, 0, buffer.length);
            if (read > 0) {
                totalRead += read;
                baos.write(buffer, 0, read);
            } else if (read < 0) {
                // EOF
                break;
            } else {
                // read == 0:
                // zero result may be returned *only* in case if the 4th
                // parameter is 0. Since in our case this is 'buffer.length',
                // zero return value clearly indicates a bug:
                throw new replacedertionError("FSDataInputStream#read(4) returned 0, while " + " the 4th method parameter is " + buffer.length + ".");
            }
        }
        final byte[] result = baos.toByteArray();
        return result;
    } finally {
        if (close) {
            fsdis.close();
        }
    }
}

19 Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB

private static void expectSeekIOE(FSDataInputStream fsdis, long seekPos, String message) {
    try {
        fsdis.seek(seekPos);
        replacedertTrue(message + " (Position = " + fsdis.getPos() + ")", false);
    } catch (IOException ioe) {
    // okay
    }
}

19 Source : TestHadoopArchives.java
with Apache License 2.0
from NJUJYB

private static byte[] readAllWithReadFully(int totalLength, FSDataInputStream fsdis, boolean close) throws IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // Simulate reading of some data structures of known length:
    final byte[] buffer = new byte[17];
    final int times = totalLength / buffer.length;
    final int remainder = totalLength % buffer.length;
    // it would be simpler to leave the position tracking to the
    // InputStream, but we need to check the methods #readFully(2)
    // and #readFully(4) that receive the position as a parameter:
    int position = 0;
    try {
        // read "data structures":
        for (int i = 0; i < times; i++) {
            fsdis.readFully(position, buffer);
            position += buffer.length;
            baos.write(buffer);
        }
        if (remainder > 0) {
            // read the remainder:
            fsdis.readFully(position, buffer, 0, remainder);
            position += remainder;
            baos.write(buffer, 0, remainder);
        }
        try {
            fsdis.readFully(position, buffer, 0, 1);
            replacedertTrue(false);
        } catch (IOException ioe) {
        // okay
        }
        replacedertEquals(totalLength, position);
        final byte[] result = baos.toByteArray();
        replacedertEquals(totalLength, result.length);
        return result;
    } finally {
        if (close) {
            fsdis.close();
        }
    }
}

19 Source : TestRpcProgramNfs3.java
with Apache License 2.0
from NJUJYB

private byte[] getFileContentsUsingDfs(String fileName, int len) throws Exception {
    final FSDataInputStream in = hdfs.open(new Path(fileName));
    final byte[] ret = new byte[len];
    in.readFully(ret);
    try {
        in.readByte();
        replacedert.fail("expected end of file");
    } catch (EOFException e) {
    // expected. Unfortunately there is no replacedociated message to check
    }
    in.close();
    return ret;
}

19 Source : DFSClientCache.java
with Apache License 2.0
from NJUJYB

FSDataInputStream getDfsInputStream(String userName, String inodePath) {
    DFSInputStreamCaheKey k = new DFSInputStreamCaheKey(userName, inodePath);
    FSDataInputStream s = null;
    try {
        s = inputstreamCache.get(k);
    } catch (ExecutionException e) {
        LOG.warn("Failed to create DFSInputStream for user:" + userName + " Cause:" + e);
    }
    return s;
}

19 Source : TestPread.java
with Apache License 2.0
from NJUJYB

private void doPread(FSDataInputStream stm, long position, byte[] buffer, int offset, int length) throws IOException {
    int nread = 0;
    long totalRead = 0;
    DFSInputStream dfstm = null;
    if (stm.getWrappedStream() instanceof DFSInputStream) {
        dfstm = (DFSInputStream) (stm.getWrappedStream());
        totalRead = dfstm.getReadStatistics().getTotalBytesRead();
    }
    while (nread < length) {
        int nbytes = stm.read(position + nread, buffer, offset + nread, length - nread);
        replacedertTrue("Error in pread", nbytes > 0);
        nread += nbytes;
    }
    if (dfstm != null) {
        if (isHedgedRead) {
            replacedertTrue("Expected read statistic to be incremented", length <= dfstm.getReadStatistics().getTotalBytesRead() - totalRead);
        } else {
            replacedertEquals("Expected read statistic to be incremented", length, dfstm.getReadStatistics().getTotalBytesRead() - totalRead);
        }
    }
}

19 Source : TestFileCreation.java
with Apache License 2.0
from NJUJYB

private byte[] readAll(FSDataInputStream in) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    byte[] buffer = new byte[1024];
    int n = 0;
    while ((n = in.read(buffer)) > -1) {
        out.write(buffer, 0, n);
    }
    return out.toByteArray();
}

19 Source : TestBlockTokenWithDFS.java
with Apache License 2.0
from NJUJYB

// read a file using blockSeekTo()
private boolean checkFile1(FSDataInputStream in) {
    byte[] toRead = new byte[FILE_SIZE];
    int totalRead = 0;
    int nRead = 0;
    try {
        while ((nRead = in.read(toRead, totalRead, toRead.length - totalRead)) > 0) {
            totalRead += nRead;
        }
    } catch (IOException e) {
        return false;
    }
    replacedertEquals("Cannot read file.", toRead.length, totalRead);
    return checkFile(toRead);
}

19 Source : AbstractContractOpenTest.java
with Apache License 2.0
from NJUJYB

/**
 * Test Seek operations
 */
public abstract clreplaced AbstractContractOpenTest extends AbstractFSContractTestBase {

    private FSDataInputStream instream;

    @Override
    protected Configuration createConfiguration() {
        Configuration conf = super.createConfiguration();
        conf.setInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, 4096);
        return conf;
    }

    @Override
    public void teardown() throws Exception {
        IOUtils.closeStream(instream);
        instream = null;
        super.teardown();
    }

    @Test
    public void testOpenReadZeroByteFile() throws Throwable {
        describe("create & read a 0 byte file");
        Path path = path("zero.txt");
        touch(getFileSystem(), path);
        instream = getFileSystem().open(path);
        replacedertEquals(0, instream.getPos());
        // expect initial read to fail
        int result = instream.read();
        replacedertMinusOne("initial byte read", result);
    }

    @Test
    public void testFsIsEncrypted() throws Exception {
        describe("create an empty file and call FileStatus.isEncrypted()");
        final Path path = path("file");
        createFile(getFileSystem(), path, false, new byte[0]);
        final FileStatus stat = getFileSystem().getFileStatus(path);
        replacedertFalse("Expecting false for stat.isEncrypted()", stat.isEncrypted());
    }

    @Test
    public void testOpenReadDir() throws Throwable {
        describe("create & read a directory");
        Path path = path("zero.dir");
        mkdirs(path);
        try {
            instream = getFileSystem().open(path);
            // at this point we've opened a directory
            fail("A directory has been opened for reading");
        } catch (FileNotFoundException e) {
            handleExpectedException(e);
        } catch (IOException e) {
            handleRelaxedException("opening a directory for reading", "FileNotFoundException", e);
        }
    }

    @Test
    public void testOpenReadDirWithChild() throws Throwable {
        describe("create & read a directory which has a child");
        Path path = path("zero.dir");
        mkdirs(path);
        Path path2 = new Path(path, "child");
        mkdirs(path2);
        try {
            instream = getFileSystem().open(path);
            // at this point we've opened a directory
            fail("A directory has been opened for reading");
        } catch (FileNotFoundException e) {
            handleExpectedException(e);
        } catch (IOException e) {
            handleRelaxedException("opening a directory for reading", "FileNotFoundException", e);
        }
    }

    @Test
    public void testOpenFileTwice() throws Throwable {
        describe("verify that two opened file streams are independent");
        Path path = path("testopenfiletwice.txt");
        byte[] block = dataset(TEST_FILE_LEN, 0, 255);
        // this file now has a simple rule: offset => value
        createFile(getFileSystem(), path, false, block);
        // open first
        FSDataInputStream instream1 = getFileSystem().open(path);
        int c = instream1.read();
        replacedertEquals(0, c);
        FSDataInputStream instream2 = null;
        try {
            instream2 = getFileSystem().open(path);
            replacedertEquals("first read of instream 2", 0, instream2.read());
            replacedertEquals("second read of instream 1", 1, instream1.read());
            instream1.close();
            replacedertEquals("second read of instream 2", 1, instream2.read());
            // close instream1 again
            instream1.close();
        } finally {
            IOUtils.closeStream(instream1);
            IOUtils.closeStream(instream2);
        }
    }

    @Test
    public void testSequentialRead() throws Throwable {
        describe("verify that sequential read() operations return values");
        Path path = path("testsequentialread.txt");
        int len = 4;
        // 64
        int base = 0x40;
        byte[] block = dataset(len, base, base + len);
        // this file now has a simple rule: offset => (value | 0x40)
        createFile(getFileSystem(), path, false, block);
        // open first
        instream = getFileSystem().open(path);
        replacedertEquals(base, instream.read());
        replacedertEquals(base + 1, instream.read());
        replacedertEquals(base + 2, instream.read());
        replacedertEquals(base + 3, instream.read());
        // and now, failures
        replacedertEquals(-1, instream.read());
        replacedertEquals(-1, instream.read());
        instream.close();
    }
}

19 Source : BoundedRangeFileInputStream.java
with Apache License 2.0
from NJUJYB

/**
 * BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop
 * FSDataInputStream as a regular input stream. One can create multiple
 * BoundedRangeFileInputStream on top of the same FSDataInputStream and they
 * would not interfere with each other.
 */
clreplaced BoundedRangeFileInputStream extends InputStream {

    private FSDataInputStream in;

    private long pos;

    private long end;

    private long mark;

    private final byte[] oneByte = new byte[1];

    /**
     * Constructor
     *
     * @param in
     *          The FSDataInputStream we connect to.
     * @param offset
     *          Begining offset of the region.
     * @param length
     *          Length of the region.
     *
     *          The actual length of the region may be smaller if (off_begin +
     *          length) goes beyond the end of FS input stream.
     */
    public BoundedRangeFileInputStream(FSDataInputStream in, long offset, long length) {
        if (offset < 0 || length < 0) {
            throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
        }
        this.in = in;
        this.pos = offset;
        this.end = offset + length;
        this.mark = -1;
    }

    @Override
    public int available() throws IOException {
        int avail = in.available();
        if (pos + avail > end) {
            avail = (int) (end - pos);
        }
        return avail;
    }

    @Override
    public int read() throws IOException {
        int ret = read(oneByte);
        if (ret == 1)
            return oneByte[0] & 0xff;
        return -1;
    }

    @Override
    public int read(byte[] b) throws IOException {
        return read(b, 0, b.length);
    }

    @Override
    public int read(byte[] b, int off, int len) throws IOException {
        if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
            throw new IndexOutOfBoundsException();
        }
        int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
        if (n == 0)
            return -1;
        int ret = 0;
        synchronized (in) {
            in.seek(pos);
            ret = in.read(b, off, n);
        }
        if (ret < 0) {
            end = pos;
            return -1;
        }
        pos += ret;
        return ret;
    }

    @Override
    public /*
   * We may skip beyond the end of the file.
   */
    long skip(long n) throws IOException {
        long len = Math.min(n, end - pos);
        pos += len;
        return len;
    }

    @Override
    public synchronized void mark(int readlimit) {
        mark = pos;
    }

    @Override
    public synchronized void reset() throws IOException {
        if (mark < 0)
            throw new IOException("Resetting to invalid mark");
        pos = mark;
    }

    @Override
    public boolean markSupported() {
        return true;
    }

    @Override
    public void close() {
        // Invalidate the state of the stream.
        in = null;
        pos = end;
        mark = -1;
    }
}

19 Source : TestReadAndSeekPageBlobAfterWrite.java
with Apache License 2.0
from naver

/**
 * Read "size" bytes of data and verify that what was read and what was written
 * are the same.
 */
private void readRandomDataAndVerify(int size) throws AzureException, IOException {
    byte[] b = new byte[size];
    FSDataInputStream stream = fs.open(PATH);
    int bytesRead = stream.read(b);
    stream.close();
    replacedertEquals(bytesRead, size);
    // compare the data read to the data written
    replacedertTrue(comparePrefix(randomData, b, size));
}

19 Source : HadoopDataInputStreamTest.java
with Apache License 2.0
from ljygz

/**
 * Tests for the {@link HadoopDataInputStream}.
 */
public clreplaced HadoopDataInputStreamTest {

    private FSDataInputStream verifyInputStream;

    private HadoopDataInputStream testInputStream;

    @Test
    public void testSeekSkip() throws IOException {
        verifyInputStream = spy(new FSDataInputStream(new SeekableByteArrayInputStream(new byte[2 * HadoopDataInputStream.MIN_SKIP_BYTES])));
        testInputStream = new HadoopDataInputStream(verifyInputStream);
        seekAndreplacedert(10);
        seekAndreplacedert(10 + HadoopDataInputStream.MIN_SKIP_BYTES + 1);
        seekAndreplacedert(testInputStream.getPos() - 1);
        seekAndreplacedert(testInputStream.getPos() + 1);
        seekAndreplacedert(testInputStream.getPos() - HadoopDataInputStream.MIN_SKIP_BYTES);
        seekAndreplacedert(testInputStream.getPos());
        seekAndreplacedert(0);
        seekAndreplacedert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES);
        seekAndreplacedert(testInputStream.getPos() + HadoopDataInputStream.MIN_SKIP_BYTES - 1);
        try {
            seekAndreplacedert(-1);
            replacedert.fail();
        } catch (Exception ignore) {
        }
        try {
            seekAndreplacedert(-HadoopDataInputStream.MIN_SKIP_BYTES - 1);
            replacedert.fail();
        } catch (Exception ignore) {
        }
    }

    private void seekAndreplacedert(long seekPos) throws IOException {
        replacedert.replacedertEquals(verifyInputStream.getPos(), testInputStream.getPos());
        long delta = seekPos - testInputStream.getPos();
        testInputStream.seek(seekPos);
        if (delta > 0L && delta <= HadoopDataInputStream.MIN_SKIP_BYTES) {
            verify(verifyInputStream, atLeastOnce()).skip(anyLong());
            verify(verifyInputStream, never()).seek(anyLong());
        } else if (delta != 0L) {
            verify(verifyInputStream, atLeastOnce()).seek(seekPos);
            verify(verifyInputStream, never()).skip(anyLong());
        } else {
            verify(verifyInputStream, never()).seek(anyLong());
            verify(verifyInputStream, never()).skip(anyLong());
        }
        replacedert.replacedertEquals(seekPos, verifyInputStream.getPos());
        reset(verifyInputStream);
    }

    private static final clreplaced SeekableByteArrayInputStream extends ByteArrayInputStreamWithPos implements Seekable, PositionedReadable {

        public SeekableByteArrayInputStream(byte[] buffer) {
            super(buffer);
        }

        @Override
        public void seek(long pos) throws IOException {
            setPosition((int) pos);
        }

        @Override
        public long getPos() throws IOException {
            return getPosition();
        }

        @Override
        public boolean seekToNewSource(long targetPos) throws IOException {
            return false;
        }

        @Override
        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public void readFully(long position, byte[] buffer) throws IOException {
            throw new UnsupportedOperationException();
        }
    }
}

19 Source : ColumnDataReader.java
with Apache License 2.0
from lealone

public clreplaced ColumnDataReader {

    static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ColumnDataReader.clreplaced);

    private final long endPosition;

    public final FSDataInputStream input;

    public ColumnDataReader(FSDataInputStream input, long start, long length) throws IOException {
        this.input = input;
        this.input.seek(start);
        this.endPosition = start + length;
    }

    public PageHeader readPageHeader() throws IOException {
        return Util.readPageHeader(input);
    }

    public FSDataInputStream getInputStream() {
        return input;
    }

    public BytesInput getPageAsBytesInput(int pageLength) throws IOException {
        byte[] b = new byte[pageLength];
        input.read(b);
        return new HadoopBytesInput(b);
    }

    public void loadPage(DrillBuf target, int pageLength) throws IOException {
        target.clear();
        HadoopStreams.wrap(input).read(target.nioBuffer(0, pageLength));
        target.writerIndex(pageLength);
    }

    public void clear() {
        try {
            input.close();
        } catch (IOException ex) {
            logger.warn("Error while closing input stream.", ex);
        }
    }

    public boolean hasRemainder() throws IOException {
        return input.getPos() < endPosition;
    }

    public clreplaced HadoopBytesInput extends BytesInput {

        private final byte[] pageBytes;

        public HadoopBytesInput(byte[] pageBytes) {
            super();
            this.pageBytes = pageBytes;
        }

        @Override
        public byte[] toByteArray() throws IOException {
            return pageBytes;
        }

        @Override
        public long size() {
            return pageBytes.length;
        }

        @Override
        public void writeAllTo(OutputStream out) throws IOException {
            out.write(pageBytes);
        }
    }
}

19 Source : FSInputGeneralColumnDataReader.java
with Apache License 2.0
from Kyligence

public clreplaced FSInputGeneralColumnDataReader implements ColumnDataReader {

    private FSDataInputStream fsInputStream;

    private int numOfVals;

    public FSInputGeneralColumnDataReader(FSDataInputStream fsInputStream, int dataStartOffset, int dataLength) throws IOException {
        this.fsInputStream = fsInputStream;
        fsInputStream.seek(dataStartOffset + dataLength - 4L);
        this.numOfVals = fsInputStream.readInt();
        fsInputStream.seek(dataStartOffset);
    }

    @Override
    public byte[] read(int index) {
        throw new UnsupportedOperationException("not support to read row operation");
    }

    @Override
    public Iterator<byte[]> iterator() {
        return new Iterator<byte[]>() {

            int readRowCount = 0;

            @Override
            public boolean hasNext() {
                return readRowCount < numOfVals;
            }

            @Override
            public byte[] next() {
                try {
                    int size = fsInputStream.readInt();
                    byte[] result = new byte[size];
                    fsInputStream.readFully(result);
                    readRowCount++;
                    return result;
                } catch (IOException e) {
                    throw new NoSuchElementException("error when read data");
                }
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException("unSupport operation");
            }
        };
    }

    @Override
    public void close() throws IOException {
        fsInputStream.close();
    }
}

19 Source : FSInputNoCompressedColumnReader.java
with Apache License 2.0
from Kyligence

public clreplaced FSInputNoCompressedColumnReader implements ColumnDataReader {

    private FSDataInputStream fsInputStream;

    private byte[] readBuffer;

    private int colDataStartOffset;

    private int colValLength;

    private int rowCount;

    public FSInputNoCompressedColumnReader(FSDataInputStream fsInputStream, int colDataStartOffset, int colValLength, int rowCount) throws IOException {
        this.fsInputStream = fsInputStream;
        this.colDataStartOffset = colDataStartOffset;
        fsInputStream.seek(colDataStartOffset);
        this.colValLength = colValLength;
        this.rowCount = rowCount;
        this.readBuffer = new byte[colValLength];
    }

    public Iterator<byte[]> iterator() {
        return new NoCompressedColumnDataItr();
    }

    @Override
    public byte[] read(int rowNum) {
        throw new UnsupportedOperationException("not support to read row operation");
    }

    @Override
    public void close() throws IOException {
        fsInputStream.close();
    }

    private clreplaced NoCompressedColumnDataItr implements Iterator<byte[]> {

        private int readRowCount = 0;

        @Override
        public boolean hasNext() {
            return readRowCount < rowCount;
        }

        @Override
        public byte[] next() {
            try {
                fsInputStream.readFully(readBuffer);
            } catch (IOException e) {
                throw new RuntimeException("error when read data", e);
            }
            readRowCount++;
            return readBuffer;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException("not supported");
        }
    }
}

19 Source : ColumnarStoreMetricsDesc.java
with Apache License 2.0
from Kyligence

public ColumnDataReader getMetricsReaderFromFSInput(FSDataInputStream inputStream, int columnDataStartOffset, int columnDataLength, int rowCount) throws IOException {
    if (Compression.LZ4 == compression && fixLen != -1) {
        return new FSInputLZ4CompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
    }
    if (fixLen != -1) {
        return new FSInputNoCompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength / rowCount, rowCount);
    }
    return new FSInputGeneralColumnDataReader(inputStream, columnDataStartOffset, columnDataLength);
}

19 Source : ColumnarStoreDimDesc.java
with Apache License 2.0
from Kyligence

public ColumnDataReader getDimReaderFromFSInput(FSDataInputStream inputStream, int columnDataStartOffset, int columnDataLength, int rowCount) throws IOException {
    if (compression == Compression.LZ4) {
        return new FSInputLZ4CompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
    } else if (compression == Compression.RUN_LENGTH) {
        return new FSInputRLECompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength, rowCount);
    }
    return new FSInputNoCompressedColumnReader(inputStream, columnDataStartOffset, columnDataLength / rowCount, rowCount);
}

19 Source : HdfsDB.java
with MIT License
from fengdis

/**
 * 下载文件
 * @param path
 * @param local
 * @throws Exception
 */
public void downLoad(String path, String local) throws Exception {
    FSDataInputStream in = fs.open(new Path(path));
    OutputStream out = new FileOutputStream(local);
    IOUtils.copyBytes(in, out, 4096, true);
}

19 Source : TestFileLink.java
with Apache License 2.0
from fengchen8086

private static void skipBuffer(FSDataInputStream in, byte v) throws IOException {
    byte[] data = new byte[8192];
    try {
        int n;
        while ((n = in.read(data)) == data.length) {
            for (int i = 0; i < data.length; ++i) {
                if (data[i] != v)
                    throw new Exception("File changed");
            }
        }
    } catch (Exception e) {
    }
}

19 Source : TestHFileBlockIndex.java
with Apache License 2.0
from fengchen8086

private void readIndex(boolean useTags) throws IOException {
    long fileSize = fs.getFileStatus(path).getLen();
    LOG.info("Size of " + path + ": " + fileSize);
    FSDataInputStream istream = fs.open(path);
    HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(true).withIncludesMvcc(includesMemstoreTS).withIncludesTags(useTags).withCompression(compr).build();
    HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(istream, fs.getFileStatus(path).getLen(), meta);
    BlockReaderWrapper brw = new BlockReaderWrapper(blockReader);
    HFileBlockIndex.BlockIndexReader indexReader = new HFileBlockIndex.BlockIndexReader(KeyValue.RAW_COMPARATOR, numLevels, brw);
    indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset, fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries);
    long prevOffset = -1;
    int i = 0;
    int expectedHitCount = 0;
    int expectedMissCount = 0;
    LOG.info("Total number of keys: " + keys.size());
    for (byte[] key : keys) {
        replacedertTrue(key != null);
        replacedertTrue(indexReader != null);
        HFileBlock b = indexReader.seekToDataBlock(new KeyValue.KeyOnlyKeyValue(key, 0, key.length), null, true, true, false, null);
        if (KeyValue.COMPARATOR.compareFlatKey(key, firstKeyInFile) < 0) {
            replacedertTrue(b == null);
            ++i;
            continue;
        }
        String keyStr = "key #" + i + ", " + Bytes.toStringBinary(key);
        replacedertTrue("seekToDataBlock failed for " + keyStr, b != null);
        if (prevOffset == b.getOffset()) {
            replacedertEquals(++expectedHitCount, brw.hitCount);
        } else {
            LOG.info("First key in a new block: " + keyStr + ", block offset: " + b.getOffset() + ")");
            replacedertTrue(b.getOffset() > prevOffset);
            replacedertEquals(++expectedMissCount, brw.missCount);
            prevOffset = b.getOffset();
        }
        ++i;
    }
    istream.close();
}

19 Source : SnapshotManifest.java
with Apache License 2.0
from fengchen8086

/*
   * Read the SnapshotDataManifest file
   */
private SnapshotDataManifest readDataManifest() throws IOException {
    FSDataInputStream in = null;
    try {
        in = fs.open(new Path(workingDir, DATA_MANIFEST_NAME));
        CodedInputStream cin = CodedInputStream.newInstance(in);
        cin.setSizeLimit(manifestSizeLimit);
        return SnapshotDataManifest.parseFrom(cin);
    } catch (FileNotFoundException e) {
        return null;
    } finally {
        if (in != null)
            in.close();
    }
}

19 Source : BoundedRangeIndexFileInputStream.java
with Apache License 2.0
from fengchen8086

/**
 * BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop
 * FSDataInputStream as a regular input stream. One can create multiple
 * BoundedRangeFileInputStream on top of the same FSDataInputStream and they
 * would not interfere with each other.
 * Copied from hadoop-335 tfile.
 */
clreplaced BoundedRangeIndexFileInputStream extends InputStream {

    private FSDataInputStream in;

    private long pos;

    private long end;

    private long mark;

    private final byte[] oneByte = new byte[1];

    private final boolean pread;

    /**
     * Constructor
     *
     * @param in
     *          The FSDataInputStream we connect to.
     * @param offset
     *          Beginning offset of the region.
     * @param length
     *          Length of the region.
     * @param pread If true, use Filesystem positional read rather than seek+read.
     *
     *          The actual length of the region may be smaller if (off_begin +
     *          length) goes beyond the end of FS input stream.
     */
    public BoundedRangeIndexFileInputStream(FSDataInputStream in, long offset, long length, final boolean pread) {
        if (offset < 0 || length < 0) {
            throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
        }
        this.in = in;
        this.pos = offset;
        this.end = offset + length;
        this.mark = -1;
        this.pread = pread;
    }

    @Override
    public int available() throws IOException {
        int avail = in.available();
        if (pos + avail > end) {
            avail = (int) (end - pos);
        }
        return avail;
    }

    @Override
    public int read() throws IOException {
        int ret = read(oneByte);
        if (ret == 1)
            return oneByte[0] & 0xff;
        return -1;
    }

    @Override
    public int read(byte[] b) throws IOException {
        return read(b, 0, b.length);
    }

    @Override
    public int read(byte[] b, int off, int len) throws IOException {
        if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
            throw new IndexOutOfBoundsException();
        }
        int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
        if (n == 0)
            return -1;
        int ret = 0;
        if (this.pread) {
            ret = in.read(pos, b, off, n);
        } else {
            synchronized (in) {
                in.seek(pos);
                ret = in.read(b, off, n);
            }
        }
        if (ret < 0) {
            end = pos;
            return -1;
        }
        pos += ret;
        return ret;
    }

    @Override
    public /*
   * We may skip beyond the end of the file.
   */
    long skip(long n) throws IOException {
        long len = Math.min(n, end - pos);
        pos += len;
        return len;
    }

    @Override
    public void mark(int readlimit) {
        mark = pos;
    }

    @Override
    public void reset() throws IOException {
        if (mark < 0)
            throw new IOException("Resetting to invalid mark");
        pos = mark;
    }

    @Override
    public boolean markSupported() {
        return true;
    }

    @Override
    public void close() {
        // Invalidate the state of the stream.
        in = null;
        pos = end;
        mark = -1;
    }
}

19 Source : HdfsSeekableInputStream.java
with MIT License
from epam

/**
 * Source:      HdfsSeekableInputStream.java
 * Created:     4/29/2016
 * Project:     CATGenome Browser
 * Make:        IntelliJ IDEA 14.1.1, JDK 1.8
 * <p>
 * A {@link SeekableStream} extension, designed to read data from HDFS
 * </p>
 */
public clreplaced HdfsSeekableInputStream extends SeekableStream {

    private FSDataInputStream stream;

    /**
     * @param stream a HDFS {@link InputStream}
     * @throws IOException
     */
    public HdfsSeekableInputStream(InputStream stream) throws IOException {
        this.stream = new FSDataInputStream(stream);
    }

    @Override
    public long length() {
        return 0;
    }

    @Override
    public long position() throws IOException {
        return stream.getPos();
    }

    @Override
    public void seek(long position) throws IOException {
        stream.seek(position);
    }

    @Override
    public int read() throws IOException {
        return stream.read();
    }

    @Override
    public int read(byte[] buffer, int offset, int length) throws IOException {
        return stream.read(buffer, offset, length);
    }

    @Override
    public void close() throws IOException {
        stream.close();
    }

    @Override
    public boolean eof() throws IOException {
        return false;
    }

    @Override
    public String getSource() {
        return null;
    }
}

19 Source : FSDataInputStreamWrapper.java
with Apache License 2.0
from dremio

public static FSInputStream of(FSDataInputStream in) throws IOException {
    if (in.getWrappedStream() instanceof ByteBufferReadable) {
        return new FSDataInputStreamWrapper(in);
    }
    return new ByteArrayFSInputStream(in);
}

19 Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio

// See DX-15492
private void openNonExistentFileInPath(org.apache.hadoop.fs.Path f) throws IOException {
    org.apache.hadoop.fs.Path nonExistentFile = f.suffix(NON_EXISTENT_FILE_SUFFIX + NON_EXISTENT_FILE_COUNTER++);
    try (FSDataInputStream is = underlyingFs.open(nonExistentFile)) {
    } catch (FileNotFoundException fileNotFoundException) {
        return;
    } catch (AccessControlException accessControlException) {
        String errMsg = "Open failed for file: " + f.toString() + " error: Permission denied (13)";
        throw new AccessControlException(errMsg);
    } catch (Exception e) {
        // Re-throw exception
        throw e;
    }
}

19 Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio

FSInputStream newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
    try {
        return (operatorStats != null) ? com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWithStatsWrapper.of(is, operatorStats, true, f.toString()) : com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWrapper.of(is);
    } catch (FSError e) {
        throw propagateFSError(e);
    }
}

19 Source : HadoopFileSystemWrapper.java
with Apache License 2.0
from dremio

FSDataInputStreamWrapper newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
    try {
        return (operatorStats != null) ? new FSDataInputStreamWithStatsWrapper(is, operatorStats) : new FSDataInputStreamWrapper(is);
    } catch (FSError e) {
        throw propagateFSError(e);
    }
}

19 Source : FSDataInputStreamWrapper.java
with Apache License 2.0
from dremio

/**
 * Wrapper around FSDataInputStream to capture {@code FSError}.
 */
clreplaced FSDataInputStreamWrapper extends FSDataInputStream {

    private FSDataInputStream underlyingIs;

    public FSDataInputStreamWrapper(FSDataInputStream in) throws IOException {
        super(new WrappedInputStream(in));
        underlyingIs = in;
    }

    protected FSDataInputStreamWrapper(FSDataInputStream in, WrappedInputStream wrapped) throws IOException {
        super(wrapped);
        underlyingIs = in;
    }

    protected FSDataInputStream getUnderlyingStream() {
        return underlyingIs;
    }

    @Override
    public synchronized void seek(long desired) throws IOException {
        try {
            underlyingIs.seek(desired);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public long getPos() throws IOException {
        try {
            return underlyingIs.getPos();
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
        try {
            return underlyingIs.read(position, buffer, offset, length);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
        try {
            underlyingIs.readFully(position, buffer, offset, length);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void readFully(long position, byte[] buffer) throws IOException {
        try {
            underlyingIs.readFully(position, buffer);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public boolean seekToNewSource(long targetPos) throws IOException {
        try {
            return underlyingIs.seekToNewSource(targetPos);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    @LimitedPrivate({ "HDFS" })
    public InputStream getWrappedStream() {
        return underlyingIs.getWrappedStream();
    }

    @Override
    public int read(ByteBuffer buf) throws IOException {
        try {
            return underlyingIs.read(buf);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    public int read(long position, ByteBuffer buf) throws IOException {
        throw new UnsupportedOperationException("positioned read with ByteBuffer not supported");
    }

    @Override
    public FileDescriptor getFileDescriptor() throws IOException {
        try {
            return underlyingIs.getFileDescriptor();
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
        try {
            underlyingIs.setReadahead(readahead);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
        try {
            underlyingIs.setDropBehind(dropBehind);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
        try {
            return underlyingIs.read(bufferPool, maxLength, opts);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void releaseBuffer(ByteBuffer buffer) {
        underlyingIs.releaseBuffer(buffer);
    }

    @Override
    public int read() throws IOException {
        try {
            return underlyingIs.read();
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public long skip(long n) throws IOException {
        try {
            return underlyingIs.skip(n);
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public int available() throws IOException {
        try {
            return underlyingIs.available();
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public void close() throws IOException {
        if (underlyingIs != null) {
            try {
                underlyingIs.close();
                underlyingIs = null;
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }
    }

    @Override
    public void mark(int readlimit) {
        underlyingIs.mark(readlimit);
    }

    @Override
    public void reset() throws IOException {
        try {
            underlyingIs.reset();
        } catch (FSError e) {
            throw HadoopFileSystemWrapper.propagateFSError(e);
        }
    }

    @Override
    public boolean markSupported() {
        return underlyingIs.markSupported();
    }

    @Override
    public void unbuffer() {
        underlyingIs.unbuffer();
    }

    /**
     * We need to wrap the FSDataInputStream inside a InputStream, because read() method in InputStream is
     * overridden in FilterInputStream (super clreplaced of FSDataInputStream) as final, so we can not override in
     * FSDataInputStreamWrapper.
     */
    static clreplaced WrappedInputStream extends InputStream implements Seekable, PositionedReadable {

        private FSDataInputStream is;

        protected WrappedInputStream(FSDataInputStream is) {
            this.is = is;
        }

        /**
         * Most of the read are going to be block reads which use {@link #read(byte[], int,
         * int)}. So not adding stats for single byte reads.
         */
        @Override
        public int read() throws IOException {
            try {
                return is.read();
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException {
            try {
                return is.read(b, off, len);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public int read(byte[] b) throws IOException {
            try {
                return is.read(b);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public long skip(long n) throws IOException {
            try {
                return is.skip(n);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public int available() throws IOException {
            try {
                return is.available();
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public void close() throws IOException {
            if (is != null) {
                try {
                    is.close();
                    is = null;
                } catch (FSError e) {
                    throw HadoopFileSystemWrapper.propagateFSError(e);
                }
            }
        }

        @Override
        public synchronized void mark(int readlimit) {
            is.mark(readlimit);
        }

        @Override
        public synchronized void reset() throws IOException {
            try {
                is.reset();
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public boolean markSupported() {
            return is.markSupported();
        }

        @Override
        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
            try {
                return is.read(position, buffer, offset, length);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
            try {
                is.readFully(position, buffer, offset, length);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public void readFully(long position, byte[] buffer) throws IOException {
            try {
                is.readFully(position, buffer);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public void seek(long pos) throws IOException {
            try {
                is.seek(pos);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public long getPos() throws IOException {
            try {
                return is.getPos();
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }

        @Override
        public boolean seekToNewSource(long targetPos) throws IOException {
            try {
                return is.seekToNewSource(targetPos);
            } catch (FSError e) {
                throw HadoopFileSystemWrapper.propagateFSError(e);
            }
        }
    }
}

19 Source : DremioHadoopFileSystemWrapper.java
with Apache License 2.0
from dremio

com.dremio.io.FSInputStream newFSDataInputStreamWrapper(Path f, final FSDataInputStream is) throws IOException {
    try {
        return (operatorStats != null) ? com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWithStatsWrapper.of(is, operatorStats, true, f.toString()) : com.dremio.exec.store.hive.exec.dfs.FSDataInputStreamWrapper.of(is);
    } catch (FSError e) {
        throw propagateFSError(e);
    }
}

19 Source : HdfsTransactionLog.java
with Apache License 2.0
from apache

clreplaced FSDataFastInputStream extends FastInputStream {

    private FSDataInputStream fis;

    public FSDataFastInputStream(FSDataInputStream fis, long chPosition) {
        // super(null, new byte[10],0,0);    // a small buffer size for testing purposes
        super(null);
        this.fis = fis;
        super.readFromStream = chPosition;
    }

    @Override
    public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
        return fis.read(readFromStream, target, offset, len);
    }

    public void seek(long position) throws IOException {
        if (position <= readFromStream && position >= getBufferPos()) {
            // seek within buffer
            pos = (int) (position - getBufferPos());
        } else {
            // long currSize = ch.size();   // not needed - underlying read should handle (unless read never done)
            // if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + " on file of size " + currSize + " file=" + ch);
            readFromStream = position;
            end = pos = 0;
        }
        replacedert position() == position;
    }

    /**
     * where is the start of the buffer relative to the whole file
     */
    public long getBufferPos() {
        return readFromStream - end;
    }

    public int getBufferSize() {
        return buf.length;
    }

    @Override
    public void close() throws IOException {
        fis.close();
    }

    @Override
    public String toString() {
        return "readFromStream=" + readFromStream + " pos=" + pos + " end=" + end + " bufferPos=" + getBufferPos() + " position=" + position();
    }
}

19 Source : AbstractContractUnbufferTest.java
with Apache License 2.0
from apache

protected void validateFullFileContents(FSDataInputStream stream) throws IOException {
    replacedertNotNull(stream);
    validateFileContents(stream, TEST_FILE_LEN, 0);
}

19 Source : HadoopUtils.java
with Apache License 2.0
from apache

/**
 * cat file on hdfs
 *
 * @param hdfsFilePath hdfs file path
 * @return byte[] byte array
 * @throws IOException errors
 */
public byte[] catFile(String hdfsFilePath) throws IOException {
    if (StringUtils.isBlank(hdfsFilePath)) {
        logger.error("hdfs file path:{} is blank", hdfsFilePath);
        return new byte[0];
    }
    try (FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath))) {
        return IOUtils.toByteArray(fsDataInputStream);
    }
}

19 Source : HadoopStreams.java
with Apache License 2.0
from apache

/**
 * Wraps a {@link FSDataInputStream} in a {@link SeekableInputStream} implementation for readers.
 *
 * @param stream a Hadoop FSDataInputStream
 * @return a SeekableInputStream
 */
static SeekableInputStream wrap(FSDataInputStream stream) {
    return new HadoopSeekableInputStream(stream);
}

19 Source : InLineFsDataInputStream.java
with Apache License 2.0
from apache

/**
 * Inline {@link FSDataInputStream}. A startOffset that is preplaceded in is replacedumed to be the start of the InputStream.
 * All operations are handled having the {@code startOffset} as starting point.
 */
public clreplaced InLineFsDataInputStream extends FSDataInputStream {

    private final int startOffset;

    private final FSDataInputStream outerStream;

    private final int length;

    public InLineFsDataInputStream(int startOffset, FSDataInputStream outerStream, int length) throws IOException {
        super(outerStream.getWrappedStream());
        this.startOffset = startOffset;
        this.outerStream = outerStream;
        this.length = length;
        outerStream.seek(startOffset);
    }

    @Override
    public void seek(long desired) throws IOException {
        if (desired > length) {
            throw new IOException("Attempting to seek past inline content");
        }
        outerStream.seek(startOffset + desired);
    }

    @Override
    public long getPos() throws IOException {
        return outerStream.getPos() - startOffset;
    }

    @Override
    public int read(long position, byte[] buffer, int offset, int length) throws IOException {
        if ((length + offset) > this.length) {
            throw new IOException("Attempting to read past inline content");
        }
        return outerStream.read(startOffset + position, buffer, offset, length);
    }

    @Override
    public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
        if ((length + offset) > this.length) {
            throw new IOException("Attempting to read past inline content");
        }
        outerStream.readFully(startOffset + position, buffer, offset, length);
    }

    @Override
    public void readFully(long position, byte[] buffer) throws IOException {
        readFully(position, buffer, 0, buffer.length);
    }

    @Override
    public boolean seekToNewSource(long targetPos) throws IOException {
        if (targetPos > this.length) {
            throw new IOException("Attempting to seek past inline content");
        }
        return outerStream.seekToNewSource(startOffset + targetPos);
    }

    @Override
    public int read(ByteBuffer buf) throws IOException {
        return outerStream.read(buf);
    }

    @Override
    public FileDescriptor getFileDescriptor() throws IOException {
        return outerStream.getFileDescriptor();
    }

    @Override
    public void setReadahead(Long readahead) throws IOException, UnsupportedOperationException {
        if (readahead > this.length) {
            throw new IOException("Attempting to set read ahead past inline content");
        }
        outerStream.setReadahead(readahead);
    }

    @Override
    public void setDropBehind(Boolean dropBehind) throws IOException, UnsupportedOperationException {
        outerStream.setDropBehind(dropBehind);
    }

    @Override
    public ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException {
        if (maxLength > this.length) {
            throw new IOException("Attempting to read max length beyond inline content");
        }
        return outerStream.read(bufferPool, maxLength, opts);
    }

    @Override
    public void releaseBuffer(ByteBuffer buffer) {
        outerStream.releaseBuffer(buffer);
    }

    @Override
    public void unbuffer() {
        outerStream.unbuffer();
    }
}

19 Source : HoodieWrapperFileSystem.java
with Apache License 2.0
from apache

private FSDataInputStream wrapInputStream(final Path path, FSDataInputStream fsDataInputStream) throws IOException {
    if (fsDataInputStream instanceof TimedFSDataInputStream) {
        return fsDataInputStream;
    }
    return new TimedFSDataInputStream(path, fsDataInputStream);
}

18 Source : ColumnChunkIncReadStore.java
with Apache License 2.0
from zpochen

public void close() throws IOException {
    for (FSDataInputStream stream : streams) {
        stream.close();
    }
    for (ColumnChunkIncPageReader reader : columns.values()) {
        reader.close();
    }
}

18 Source : TextInput.java
with Apache License 2.0
from zpochen

/**
 * Clreplaced that fronts an InputStream to provide a byte consumption interface.
 * Also manages only reading lines to and from each split.
 */
final clreplaced TextInput {

    private final byte[] lineSeparator;

    private final byte normalizedLineSeparator;

    private final TextParsingSettings settings;

    private long lineCount;

    private long charCount;

    /**
     * The starting position in the file.
     */
    private final long startPos;

    private final long endPos;

    private int bufferMark;

    private long streamMark;

    private long streamPos;

    private final Seekable seekable;

    private final FSDataInputStream inputFS;

    private final InputStream input;

    private final DrillBuf buffer;

    private final ByteBuffer underlyingBuffer;

    private final long bStart;

    private final long bStartMinus1;

    private final boolean bufferReadable;

    /**
     * Whether there was a possible partial line separator on the previous
     * read so we dropped it and it should be appended to next read.
     */
    private int remByte = -1;

    /**
     * The current position in the buffer.
     */
    public int bufferPtr;

    /**
     * The quanreplacedy of valid data in the buffer.
     */
    public int length = -1;

    private boolean endFound = false;

    /**
     * Creates a new instance with the mandatory characters for handling newlines transparently.
     * lineSeparator the sequence of characters that represent a newline, as defined in {@link Format#getLineSeparator()}
     * normalizedLineSeparator the normalized newline character (as defined in {@link Format#getNormalizedNewline()}) that is used to replace any lineSeparator sequence found in the input.
     */
    public TextInput(TextParsingSettings settings, InputStream input, DrillBuf readBuffer, long startPos, long endPos) {
        this.lineSeparator = settings.getNewLineDelimiter();
        byte normalizedLineSeparator = settings.getNormalizedNewLine();
        Preconditions.checkArgument(input instanceof Seekable, "Text input only supports an InputStream that supports Seekable.");
        boolean isCompressed = input instanceof CompressionInputStream;
        Preconditions.checkArgument(!isCompressed || startPos == 0, "Cannot use split on compressed stream.");
        // splits aren't allowed with compressed data.  The split length will be the compressed size which means we'll normally end prematurely.
        if (isCompressed && endPos > 0) {
            endPos = Long.MAX_VALUE;
        }
        this.input = input;
        this.seekable = (Seekable) input;
        this.settings = settings;
        if (input instanceof FSDataInputStream) {
            this.inputFS = (FSDataInputStream) input;
            this.bufferReadable = inputFS.getWrappedStream() instanceof ByteBufferReadable;
        } else {
            this.inputFS = null;
            this.bufferReadable = false;
        }
        this.startPos = startPos;
        this.endPos = endPos;
        this.normalizedLineSeparator = normalizedLineSeparator;
        this.buffer = readBuffer;
        this.bStart = buffer.memoryAddress();
        this.bStartMinus1 = bStart - 1;
        this.underlyingBuffer = buffer.nioBuffer(0, buffer.capacity());
    }

    /**
     * Test the input to position for read start.  If the input is a non-zero split or
     * splitFirstLine is enabled, input will move to appropriate complete line.
     * @throws IOException
     */
    final void start() throws IOException {
        lineCount = 0;
        if (startPos > 0) {
            seekable.seek(startPos);
        }
        updateBuffer();
        if (length > 0) {
            if (startPos > 0 || settings.isSkipFirstLine()) {
                // move to next full record.
                skipLines(1);
            }
        }
    }

    /**
     * Helper method to get the most recent characters consumed since the last record started.
     * May get an incomplete string since we don't support stream rewind.  Returns empty string for now.
     * @return String of last few bytes.
     * @throws IOException
     */
    public String getStringSinceMarkForError() throws IOException {
        return " ";
    }

    long getPos() {
        return streamPos + bufferPtr;
    }

    public void mark() {
        streamMark = streamPos;
        bufferMark = bufferPtr;
    }

    /**
     * read some more bytes from the stream.  Uses the zero copy interface if available.  Otherwise, does byte copy.
     * @throws IOException
     */
    private void read() throws IOException {
        if (bufferReadable) {
            if (remByte != -1) {
                for (int i = 0; i <= remByte; i++) {
                    underlyingBuffer.put(lineSeparator[i]);
                }
                remByte = -1;
            }
            length = inputFS.read(underlyingBuffer);
        } else {
            byte[] b = new byte[underlyingBuffer.capacity()];
            if (remByte != -1) {
                int remBytesNum = remByte + 1;
                System.arraycopy(lineSeparator, 0, b, 0, remBytesNum);
                length = input.read(b, remBytesNum, b.length - remBytesNum);
                remByte = -1;
            } else {
                length = input.read(b);
            }
            underlyingBuffer.put(b);
        }
    }

    /**
     * Read more data into the buffer.  Will also manage split end conditions.
     * @throws IOException
     */
    private void updateBuffer() throws IOException {
        streamPos = seekable.getPos();
        underlyingBuffer.clear();
        if (endFound) {
            length = -1;
            return;
        }
        read();
        // check our data read allowance.
        if (streamPos + length >= this.endPos) {
            updateLengthBasedOnConstraint();
        }
        charCount += bufferPtr;
        bufferPtr = 1;
        buffer.writerIndex(underlyingBuffer.limit());
        buffer.readerIndex(underlyingBuffer.position());
    }

    /**
     * Checks to see if we can go over the end of our bytes constraint on the data.  If so,
     * adjusts so that we can only read to the last character of the first line that crosses
     * the split boundary.
     */
    private void updateLengthBasedOnConstraint() {
        final long max = bStart + length;
        for (long m = bStart + (endPos - streamPos); m < max; m++) {
            for (int i = 0; i < lineSeparator.length; i++) {
                long mPlus = m + i;
                if (mPlus < max) {
                    // we found a line separator and don't need to consult the next byte.
                    if (lineSeparator[i] == PlatformDependent.getByte(mPlus) && i == lineSeparator.length - 1) {
                        length = (int) (mPlus - bStart) + 1;
                        endFound = true;
                        return;
                    }
                } else {
                    // the last N characters of the read were remnant bytes. We'll hold off on dealing with these bytes until the next read.
                    remByte = i;
                    length = length - i;
                    return;
                }
            }
        }
    }

    /**
     * Get next byte from stream.  Also maintains the current line count.  Will throw a StreamFinishedPseudoException
     * when the stream has run out of bytes.
     * @return next byte from stream.
     * @throws IOException
     */
    public final byte nextChar() throws IOException {
        byte byteChar = nextCharNoNewLineCheck();
        int bufferPtrTemp = bufferPtr - 1;
        if (byteChar == lineSeparator[0]) {
            for (int i = 1; i < lineSeparator.length; i++, bufferPtrTemp++) {
                if (lineSeparator[i] != buffer.getByte(bufferPtrTemp)) {
                    return byteChar;
                }
            }
            lineCount++;
            byteChar = normalizedLineSeparator;
            // we don't need to update buffer position if line separator is one byte long
            if (lineSeparator.length > 1) {
                bufferPtr += (lineSeparator.length - 1);
                if (bufferPtr >= length) {
                    if (length != -1) {
                        updateBuffer();
                    } else {
                        throw StreamFinishedPseudoException.INSTANCE;
                    }
                }
            }
        }
        return byteChar;
    }

    /**
     * Get next byte from stream.  Do no maintain any line count  Will throw a StreamFinishedPseudoException
     * when the stream has run out of bytes.
     * @return next byte from stream.
     * @throws IOException
     */
    public final byte nextCharNoNewLineCheck() throws IOException {
        if (length == -1) {
            throw StreamFinishedPseudoException.INSTANCE;
        }
        if (BoundsChecking.BOUNDS_CHECKING_ENABLED) {
            buffer.checkBytes(bufferPtr - 1, bufferPtr);
        }
        byte byteChar = PlatformDependent.getByte(bStartMinus1 + bufferPtr);
        if (bufferPtr >= length) {
            if (length != -1) {
                updateBuffer();
                bufferPtr--;
            } else {
                throw StreamFinishedPseudoException.INSTANCE;
            }
        }
        bufferPtr++;
        return byteChar;
    }

    /**
     * Number of lines read since the start of this split.
     * @return
     */
    public final long lineCount() {
        return lineCount;
    }

    /**
     * Skip forward the number of line delimiters.  If you are in the middle of a line,
     * a value of 1 will skip to the start of the next record.
     * @param lines Number of lines to skip.
     * @throws IOException
     */
    public final void skipLines(int lines) throws IOException {
        if (lines < 1) {
            return;
        }
        long expectedLineCount = this.lineCount + lines;
        try {
            do {
                nextChar();
            } while (lineCount < expectedLineCount);
            if (lineCount < lines) {
                throw new IllegalArgumentException("Unable to skip " + lines + " lines from line " + (expectedLineCount - lines) + ". End of input reached");
            }
        } catch (EOFException ex) {
            throw new IllegalArgumentException("Unable to skip " + lines + " lines from line " + (expectedLineCount - lines) + ". End of input reached");
        }
    }

    public final long charCount() {
        return charCount + bufferPtr;
    }

    public long getLineCount() {
        return lineCount;
    }

    public void close() throws IOException {
        input.close();
    }
}

18 Source : DFSUtil.java
with Apache License 2.0
from wgzhao

// 判断file是否是RC file
private boolean isRCFile(String filepath, FSDataInputStream in) {
    // The first version of RCFile used the sequence file header.
    final byte[] originalMagic = { (byte) 'S', (byte) 'E', (byte) 'Q' };
    // The 'magic' bytes at the beginning of the RCFile
    final byte[] rcMagic = { (byte) 'R', (byte) 'C', (byte) 'F' };
    // the version that was included with the original magic, which is mapped
    // into ORIGINAL_VERSION
    final byte ORIGINAL_MAGIC_VERSION_WITH_METADATA = 6;
    // All of the versions should be place in this list.
    // version with SEQ
    final int ORIGINAL_VERSION = 0;
    // version with RCF
    // final int NEW_MAGIC_VERSION = 1
    // final int CURRENT_VERSION = NEW_MAGIC_VERSION
    final int CURRENT_VERSION = 1;
    byte version;
    byte[] magic = new byte[rcMagic.length];
    try {
        in.seek(0);
        in.readFully(magic);
        if (Arrays.equals(magic, originalMagic)) {
            byte vers = in.readByte();
            if (vers != ORIGINAL_MAGIC_VERSION_WITH_METADATA) {
                return false;
            }
            version = ORIGINAL_VERSION;
        } else {
            if (!Arrays.equals(magic, rcMagic)) {
                return false;
            }
            // Set 'version'
            version = in.readByte();
            if (version > CURRENT_VERSION) {
                return false;
            }
        }
        if (version == ORIGINAL_VERSION) {
            try {
                Clreplaced<?> keyCls = hadoopConf.getClreplacedByName(Text.readString(in));
                Clreplaced<?> valCls = hadoopConf.getClreplacedByName(Text.readString(in));
                if (!keyCls.equals(RCFile.KeyBuffer.clreplaced) || !valCls.equals(RCFile.ValueBuffer.clreplaced)) {
                    return false;
                }
            } catch (ClreplacedNotFoundException e) {
                return false;
            }
        }
        // boolean decompress = in.readBoolean(); // is compressed?
        if (version == ORIGINAL_VERSION) {
            // is block-compressed? it should be always false.
            boolean blkCompressed = in.readBoolean();
            return !blkCompressed;
        }
        return true;
    } catch (IOException e) {
        LOG.info("检查文件类型: [{}] 不是RC File.", filepath);
    }
    return false;
}

18 Source : DFSUtil.java
with Apache License 2.0
from wgzhao

// 判断是否为parquet
private boolean isParquetFile(Path file, FSDataInputStream in) {
    try {
        GroupReadSupport readSupport = new GroupReadSupport();
        ParquetReader.Builder<Group> reader = ParquetReader.builder(readSupport, file);
        ParquetReader<Group> build = reader.build();
        if (build.read() != null) {
            return true;
        }
    } catch (IOException e) {
        LOG.info("检查文件类型: [{}] 不是Parquet File.", file);
    }
    return false;
}

18 Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb

@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.clreplaced, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_FORBIDDEN + ".*")
public void testReadForbidden() throws Exception {
    try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_FORBIDDEN);
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        }
    }
}

18 Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb

@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.clreplaced, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_NOT_FOUND + ".*")
public void testReadNotFound() throws Exception {
    try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_NOT_FOUND);
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        }
    }
}

18 Source : TestTrinoS3FileSystem.java
with Apache License 2.0
from trinodb

@Test
public void testReadRequestRangeNotSatisfiable() throws Exception {
    try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_RANGE_NOT_SATISFIABLE);
        fs.initialize(new URI("s3n://test-bucket/"), new Configuration(false));
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            replacedertEquals(inputStream.read(), -1);
        }
    }
}

18 Source : FSDataInputStreamTail.java
with Apache License 2.0
from trinodb

public static long readTailForFileSize(String path, long paddedFileSize, FSDataInputStream inputStream) throws IOException {
    long position = max(paddedFileSize - MAX_SUPPORTED_PADDING_BYTES, 0);
    long maxEOFAt = paddedFileSize + 1;
    long startPos = inputStream.getPos();
    try {
        inputStream.seek(position);
        int c;
        while (position < maxEOFAt) {
            c = inputStream.read();
            if (c < 0) {
                return position;
            }
            position++;
        }
        throw rejectInvalidFileSize(path, paddedFileSize);
    } finally {
        inputStream.seek(startPos);
    }
}

See More Examples