org.apache.hadoop.fs.FSDataOutputStream.close()

Here are the examples of the java api org.apache.hadoop.fs.FSDataOutputStream.close() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

516 Examples 7

19 Source : BatchGroup.java
with Apache License 2.0
from zpochen

public void closeOutputStream() throws IOException {
    if (outputStream != null) {
        outputStream.close();
    }
}

19 Source : GenericSourceSink.java
with Apache License 2.0
from snuspl

/**
 * finish bundle.
 * @param c             context
 * @throws IOException  output stream exception
 */
@FinishBundle
public void finishBundle(final FinishBundleContext c) throws IOException {
    outputStream.close();
}

19 Source : FileSystemNodeLabelsStore.java
with Apache License 2.0
from NJUJYB

private void ensureCloseEditlogFile() throws IOException {
    editlogOs.close();
}

19 Source : FSPartFileCollection.java
with Apache License 2.0
from lealone

private void turnover() throws IOException {
    partOutStr.close();
    partOutStr = null;
    writeIndex();
    syncRequested = false;
}

19 Source : VectorizedHashAggDiskPartition.java
with Apache License 2.0
from dremio

/**
 * Once {@link VectorizedHashAggOperator} has consumed all data, it flushes
 * the in-memory data (if any) for all spilled parreplacedions and that's when we
 * finally close the output stream for parreplacedion's spill file since that is the
 * last moment we will spill anything for that parreplacedion as later on the parreplacedion
 * will transition to read-only (pure disk based) mode.
 * @throws Exception
 */
public void closeSpillStream() throws Exception {
    outputStream.close();
    outputStream = null;
}

19 Source : GenericSourceSink.java
with Apache License 2.0
from apache

/**
 * Teardown.
 *
 * @throws IOException output stream exception
 */
@Teardown
public void tearDown() throws IOException {
    outputStream.close();
}

19 Source : HoodieLogFormatWriter.java
with Apache License 2.0
from apache

private void closeStream() throws IOException {
    if (output != null) {
        flush();
        output.close();
        output = null;
        closed = true;
    }
}

18 Source : JsonRecordWriter.java
with Apache License 2.0
from zpochen

@Override
public void cleanup() throws IOException {
    gen.flush();
    stream.close();
}

18 Source : HDFSSequenceFile.java
with MIT License
from TranswarpCN

@Override
public void close() throws IOException {
    writer.close();
    outStream.close();
    unregisterCurrentStream();
}

18 Source : TestTFileUnsortedByteArrays.java
with Apache License 2.0
from NJUJYB

private void closeOutput() throws IOException {
    if (writer != null) {
        writer.close();
        writer = null;
        out.close();
        out = null;
    }
}

18 Source : TestTFileStreams.java
with Apache License 2.0
from NJUJYB

private void closeOutput() throws IOException {
    if (writer != null) {
        writer.close();
        writer = null;
    }
    if (out != null) {
        out.close();
        out = null;
    }
}

18 Source : HadoopRecoverableFsDataOutputStream.java
with Apache License 2.0
from ljygz

@Override
public void close() throws IOException {
    out.close();
}

18 Source : StreamWriterBase.java
with Apache License 2.0
from ljygz

@Override
public void close() throws IOException {
    if (outStream != null) {
        flush();
        outStream.close();
        outStream = null;
    }
}

18 Source : FSJsonLineFile.java
with Apache License 2.0
from lealone

@Override
public void close() throws IOException {
    os.close();
}

18 Source : AbstractHFileWriter.java
with Apache License 2.0
from fengchen8086

/**
 * Sets remaining trailer fields, writes the trailer to disk, and optionally
 * closes the output stream.
 */
protected void finishClose(FixedFileTrailer trailer) throws IOException {
    trailer.setMetaIndexCount(metaNames.size());
    trailer.setTotalUncompressedBytes(totalUncompressedBytes + trailer.getTrailerSize());
    trailer.setEntryCount(entryCount);
    trailer.setCompressionCodec(hFileContext.getCompression());
    trailer.serialize(outputStream);
    if (closeOutputStream) {
        outputStream.close();
        outputStream = null;
    }
}

18 Source : DirtyDataManager.java
with Apache License 2.0
from DTStack

public void close() {
    if (stream != null) {
        try {
            stream.flush();
            stream.close();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

18 Source : FSDataOutputStreamWrapper.java
with Apache License 2.0
from dremio

@Override
public void close() throws IOException {
    try {
        underlyingOS.close();
    } catch (FSError e) {
        throw propagateFSError(e);
    }
}

18 Source : FilterFSOutputStream.java
with Apache License 2.0
from dremio

/**
 * @throws IOException
 * @see java.io.OutputStream#close()
 */
@Override
public void close() throws IOException {
    out.close();
}

18 Source : ArchiveHdfsLogWriter.java
with Apache License 2.0
from apache

@Override
public void rollWriter() throws IOException {
    if (lastRollRecordId != currentRecordId) {
        // close old file
        os.close();
        hdfs.rename(currentPath, new Path(LogServiceUtils.getRolledPathForArchiveWriter(currentPath, currentRecordId)));
        lastRollRecordId = currentRecordId;
        // create new file
        os = hdfs.create(currentPath, true);
    }
}

18 Source : ArchiveHdfsLogWriter.java
with Apache License 2.0
from apache

@Override
public void close() throws IOException {
    os.close();
    if (lastRollRecordId != currentRecordId) {
        hdfs.rename(currentPath, new Path(currentPath + "_recordId_" + currentRecordId));
    }
}

18 Source : HDFSOutput.java
with Apache License 2.0
from apache

@Override
public void close() throws IOException {
    fsDataOutputStream.close();
}

17 Source : TestRubixCaching.java
with Apache License 2.0
from trinodb

private void writeFile(FSDataOutputStream outputStream, byte[] content) throws IOException {
    try {
        outputStream.write(content);
    } finally {
        outputStream.close();
    }
}

17 Source : GenericSourceSink.java
with Apache License 2.0
from snuspl

/**
 * process element.
 * @param c          context {@link ProcessContext}
 * @throws Exception exception.
 */
@ProcessElement
public void processElement(final ProcessContext c) throws Exception {
    try {
        outputStream.writeBytes(c.element() + "\n");
    } catch (Exception e) {
        outputStream.close();
        fileSystem.delete(fileName, true);
        throw new RuntimeException(e);
    }
}

17 Source : NNBenchWithoutMR.java
with Apache License 2.0
from NJUJYB

/**
 * Create and write to a given number of files.  Repeat each remote
 * operation until is suceeds (does not throw an exception).
 *
 * @return the number of exceptions caught
 */
static int createWrite() {
    int totalExceptions = 0;
    FSDataOutputStream out = null;
    boolean success;
    for (int index = 0; index < numFiles; index++) {
        int singleFileExceptions = 0;
        do {
            // create file until is succeeds or max exceptions reached
            try {
                out = fileSys.create(new Path(taskDir, "" + index), false, 512, (short) 1, bytesPerBlock);
                success = true;
            } catch (IOException ioe) {
                success = false;
                totalExceptions++;
                handleException("creating file #" + index, ioe, ++singleFileExceptions);
            }
        } while (!success);
        long toBeWritten = bytesPerFile;
        while (toBeWritten > 0) {
            int nbytes = (int) Math.min(buffer.length, toBeWritten);
            toBeWritten -= nbytes;
            try {
                // only try once
                out.write(buffer, 0, nbytes);
            } catch (IOException ioe) {
                totalExceptions++;
                handleException("writing to file #" + index, ioe, ++singleFileExceptions);
            }
        }
        do {
            // close file until is succeeds
            try {
                out.close();
                success = true;
            } catch (IOException ioe) {
                success = false;
                totalExceptions++;
                handleException("closing file #" + index, ioe, ++singleFileExceptions);
            }
        } while (!success);
    }
    return totalExceptions;
}

17 Source : EventWriter.java
with Apache License 2.0
from NJUJYB

void close() throws IOException {
    try {
        encoder.flush();
        out.close();
        out = null;
    } finally {
        IOUtils.cleanup(LOG, out);
    }
}

17 Source : TestFileAppend3.java
with Apache License 2.0
from NJUJYB

// Do small appends.
void doSmallAppends(Path file, DistributedFileSystem fs, int iterations) throws IOException {
    for (int i = 0; i < iterations; i++) {
        FSDataOutputStream stm;
        try {
            stm = fs.append(file);
        } catch (IOException e) {
            // If another thread is already appending, skip this time.
            continue;
        }
        // Failure in write or close will be terminal.
        AppendTestUtil.write(stm, 0, 123);
        stm.close();
    }
}

17 Source : TestTFileByteArrays.java
with Apache License 2.0
from NJUJYB

@Test
public void testFailureOpenEmptyFile() throws IOException {
    if (skip)
        return;
    closeOutput();
    // create an absolutely empty file
    path = new Path(fs.getWorkingDirectory(), outputFile);
    out = fs.create(path);
    out.close();
    try {
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
        replacedert.fail("Error on handling empty files.");
    } catch (EOFException e) {
    // noop, expecting exceptions
    }
}

17 Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB

@Test
public void testCorruptedChecksum() throws Exception {
    Path testPath = new Path(TEST_ROOT_DIR, "testCorruptChecksum");
    Path checksumPath = localFs.getChecksumFile(testPath);
    // write a file to generate checksum
    FSDataOutputStream out = localFs.create(testPath, true);
    out.write("testing 1 2 3".getBytes());
    out.close();
    replacedertTrue(localFs.exists(checksumPath));
    FileStatus stat = localFs.getFileStatus(checksumPath);
    // alter file directly so checksum is invalid
    out = localFs.getRawFileSystem().create(testPath, true);
    out.write("testing stale checksum".getBytes());
    out.close();
    replacedertTrue(localFs.exists(checksumPath));
    // checksum didn't change on disk
    replacedertEquals(stat, localFs.getFileStatus(checksumPath));
    Exception e = null;
    try {
        localFs.setVerifyChecksum(true);
        readFile(localFs, testPath, 1024);
    } catch (ChecksumException ce) {
        e = ce;
    } finally {
        replacedertNotNull("got checksum error", e);
    }
    localFs.setVerifyChecksum(false);
    String str = readFile(localFs, testPath, 1024);
    replacedertEquals("testing stale checksum", str);
}

17 Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB

@Test
public void testVerifyChecksum() throws Exception {
    Path testPath = new Path(TEST_ROOT_DIR, "testPath");
    Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
    FSDataOutputStream fout = localFs.create(testPath);
    fout.write("testing".getBytes());
    fout.close();
    fout = localFs.create(testPath11);
    fout.write("testing you".getBytes());
    fout.close();
    // Exercise some boundary cases - a divisor of the chunk size
    // the chunk size, 2x chunk size, and +/-1 around these.
    readFile(localFs, testPath, 128);
    readFile(localFs, testPath, 511);
    readFile(localFs, testPath, 512);
    readFile(localFs, testPath, 513);
    readFile(localFs, testPath, 1023);
    readFile(localFs, testPath, 1024);
    readFile(localFs, testPath, 1025);
    localFs.delete(localFs.getChecksumFile(testPath), true);
    replacedertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
    // copying the wrong checksum file
    FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, localFs.getChecksumFile(testPath), false, true, localFs.getConf());
    replacedertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
    boolean errorRead = false;
    try {
        readFile(localFs, testPath, 1024);
    } catch (ChecksumException ie) {
        errorRead = true;
    }
    replacedertTrue("error reading", errorRead);
    // now setting verify false, the read should succeed
    localFs.setVerifyChecksum(false);
    String str = readFile(localFs, testPath, 1024).toString();
    replacedertTrue("read", "testing".equals(str));
}

17 Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB

@Test
public void testMultiChunkFile() throws Exception {
    Path testPath = new Path(TEST_ROOT_DIR, "testMultiChunk");
    FSDataOutputStream fout = localFs.create(testPath);
    for (int i = 0; i < 1000; i++) {
        fout.write(("testing" + i).getBytes());
    }
    fout.close();
    // Exercise some boundary cases - a divisor of the chunk size
    // the chunk size, 2x chunk size, and +/-1 around these.
    readFile(localFs, testPath, 128);
    readFile(localFs, testPath, 511);
    readFile(localFs, testPath, 512);
    readFile(localFs, testPath, 513);
    readFile(localFs, testPath, 1023);
    readFile(localFs, testPath, 1024);
    readFile(localFs, testPath, 1025);
}

17 Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB

@Test
public void testAppendMissingTarget() throws Throwable {
    try {
        FSDataOutputStream out = getFileSystem().append(target);
        // got here: trouble
        out.close();
        fail("expected a failure");
    } catch (Exception e) {
        // expected
        handleExpectedException(e);
    }
}

17 Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB

@Test
public void testAppendNonexistentFile() throws Throwable {
    try {
        FSDataOutputStream out = getFileSystem().append(target);
        // got here: trouble
        out.close();
        fail("expected a failure");
    } catch (Exception e) {
        // expected
        handleExpectedException(e);
    }
}

17 Source : HiveProducer.java
with Apache License 2.0
from Kyligence

private void closeFout() {
    if (fout != null) {
        try {
            fout.close();
        } catch (Exception e) {
            logger.error("Close the path: " + curParreplacedionContentPath + " failed", e);
            if (fs instanceof DistributedFileSystem) {
                DistributedFileSystem hdfs = (DistributedFileSystem) fs;
                try {
                    boolean recovered = hdfs.recoverLease(curParreplacedionContentPath);
                } catch (Exception e1) {
                    logger.error("Recover lease for path: " + curParreplacedionContentPath + " failed", e1);
                }
            }
        }
    }
    fout = null;
}

17 Source : DataFile.java
with Apache License 2.0
from didi

public boolean close() {
    boolean flag = false;
    closeInvokedCount++;
    try {
        flush();
        stream.close();
        flag = true;
        closed = true;
    } catch (Exception e) {
        LOGGER.error("Close file error, dir:" + dir + ", file:" + fileName, e);
    }
    if (flag) {
        dir.deleteDataFile(pid);
        LOGGER.info("Close file success,file:" + getPath());
    } else {
        LOGGER.info("Close file failed, then add CloseFailStream,file:" + getPath());
        dataFileManager.addCloseFailedStream(this);
    }
    return flag;
}

17 Source : DataFile.java
with Apache License 2.0
from didi

public boolean directClose() {
    boolean flag = false;
    closeInvokedCount++;
    try {
        flush();
        stream.close();
        flag = true;
        closed = true;
    } catch (Exception e) {
        LOGGER.error("Close file error, dir:" + dir + ", file:" + fileName, e);
    }
    if (flag) {
        LOGGER.info("Close file success,file:" + getPath());
    } else {
        LOGGER.info("Close file failed, then add CloseFailStream,file:" + getPath());
        dataFileManager.addCloseFailedStream(this);
    }
    dir.deleteDataFile(pid);
    return flag;
}

17 Source : HdfsTransactionLog.java
with Apache License 2.0
from apache

private void doCloseOutput() throws IOException {
    synchronized (this) {
        if (fos == null)
            return;
        if (debug) {
            log.debug("Closing output for {}", tlogFile);
        }
        fos.flushBuffer();
        finalLogSize = fos.size();
        fos = null;
    }
    tlogOutStream.hflush();
    tlogOutStream.close();
    tlogOutStream = null;
}

17 Source : GenericSourceSink.java
with Apache License 2.0
from apache

/**
 * process element.
 *
 * @param c context {@link ProcessContext}
 * @throws Exception exception.
 */
@ProcessElement
public void processElement(final ProcessContext c) throws DataSourceException, IOException {
    try {
        outputStream.writeBytes(c.element() + "\n");
    } catch (Exception e) {
        outputStream.close();
        fileSystem.delete(fileName, true);
        throw new DataSourceException(new Exception("Processing data from source failed " + e));
    }
}

16 Source : FileSystemNodeLabelsStore.java
with Apache License 2.0
from NJUJYB

@Override
public void close() throws IOException {
    try {
        fs.close();
        editlogOs.close();
    } catch (IOException e) {
        LOG.warn("Exception happened whiling shutting down,", e);
    }
}

16 Source : TestSwiftFileSystemExtendedContract.java
with Apache License 2.0
from NJUJYB

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreateFile() throws Exception {
    final Path f = new Path("/test/testCreateFile");
    final FSDataOutputStream fsDataOutputStream = fs.create(f);
    fsDataOutputStream.close();
    replacedertExists("created file", f);
}

16 Source : SwiftFileSystemBaseTest.java
with Apache License 2.0
from NJUJYB

/**
 * Create and then close a file
 * @param path path to create
 * @throws IOException on a failure
 */
protected void createEmptyFile(Path path) throws IOException {
    FSDataOutputStream out = fs.create(path);
    out.close();
}

16 Source : TestUserResolve.java
with Apache License 2.0
from NJUJYB

/**
 * Creates users file with the content as the String usersFileContent.
 * @param usersFilePath    the path to the file that is to be created
 * @param usersFileContent Content of users file
 * @throws IOException
 */
private static void writeUserList(Path usersFilePath, String usersFileContent) throws IOException {
    FSDataOutputStream out = null;
    try {
        out = fs.create(usersFilePath, true);
        out.writeBytes(usersFileContent);
    } finally {
        if (out != null) {
            out.close();
        }
    }
}

16 Source : TestFileConcurrentReader.java
with Apache License 2.0
from NJUJYB

/**
 * Test that that writes to an incomplete block are available to a reader
 */
@Test(timeout = 30000)
public void testUnfinishedBlockRead() throws IOException {
    // create a new file in the root, write data, do no close
    Path file1 = new Path("/unfinished-block");
    FSDataOutputStream stm = TestFileCreation.createFile(fileSystem, file1, 1);
    // write partial block and sync
    int partialBlockSize = blockSize / 2;
    writeFileAndSync(stm, partialBlockSize);
    // Make sure a client can read it before it is closed
    checkCanRead(fileSystem, file1, partialBlockSize);
    stm.close();
}

16 Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB

@Test
public void testAppendToExistingFile() throws Throwable {
    byte[] original = dataset(8192, 'A', 'Z');
    byte[] appended = dataset(8192, '0', '9');
    createFile(getFileSystem(), target, false, original);
    FSDataOutputStream outputStream = getFileSystem().append(target);
    outputStream.write(appended);
    outputStream.close();
    byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), target, original.length + appended.length);
    ContractTestUtils.validateFileContent(bytes, new byte[][] { original, appended });
}

16 Source : TestDecommission.java
with Apache License 2.0
from naver

private void writeConfigFile(Path name, List<String> nodes) throws IOException {
    // delete if it already exists
    if (localFileSys.exists(name)) {
        localFileSys.delete(name, true);
    }
    FSDataOutputStream stm = localFileSys.create(name);
    if (nodes != null) {
        for (Iterator<String> it = nodes.iterator(); it.hasNext(); ) {
            String node = it.next();
            stm.writeBytes(node);
            stm.writeBytes("\n");
        }
    }
    stm.close();
}

16 Source : ContinuousFileProcessingITCase.java
with Apache License 2.0
from ljygz

/**
 * Create a file and fill it with content.
 */
private Tuple2<org.apache.hadoop.fs.Path, String> fillWithData(String base, String fileName, int fileIdx, String sampleLine) throws IOException, InterruptedException {
    replacedert (hdfs != null);
    org.apache.hadoop.fs.Path tmp = new org.apache.hadoop.fs.Path(base + "/." + fileName + fileIdx);
    FSDataOutputStream stream = hdfs.create(tmp);
    StringBuilder str = new StringBuilder();
    for (int i = 0; i < LINES_PER_FILE; i++) {
        String line = fileIdx + ": " + sampleLine + " " + i + "\n";
        str.append(line);
        stream.write(line.getBytes(ConfigConstants.DEFAULT_CHARSET));
    }
    stream.close();
    return new Tuple2<>(tmp, str.toString());
}

16 Source : FSPartFileCollection.java
with Apache License 2.0
from lealone

public void teardown() {
    logger.info("Closing hdfs part collection.");
    try {
        if (metaOs != null) {
            metaOs.close();
        }
        if (partOutStr != null) {
            logger.debug("Closing part file");
            partOutStr.close();
            if (indexOutStr != null) {
                writeIndex();
            }
        }
        if (indexOutStr != null) {
            writeIndexEnd();
            indexOutStr.close();
        }
        fs.close();
    } catch (IOException ex) {
        logger.error(ex.toString());
    }
}

16 Source : TestCmdlet.java
with Apache License 2.0
from Intel-bigdata

private void generateTestFiles() throws IOException {
    // New dir
    Path dir = new Path("/testMoveFile");
    dfs.mkdirs(dir);
    // Move to SSD
    dfs.setStoragePolicy(dir, "HOT");
    final FSDataOutputStream out1 = dfs.create(new Path("/testMoveFile/file1"), true, 1024);
    out1.writeChars("/testMoveFile/file1");
    out1.close();
    // Move to Archive
    final FSDataOutputStream out2 = dfs.create(new Path("/testMoveFile/file2"), true, 1024);
    out2.writeChars("/testMoveFile/file2");
    out2.close();
    // Move to CacheObject
    Path dir3 = new Path("/testCacheFile");
    dfs.mkdirs(dir3);
}

16 Source : TestRenameFileAction.java
with Apache License 2.0
from Intel-bigdata

@Test
public void testDestExistingFileRename() throws Exception {
    final String srcPath = "/testRename";
    final String destPath = "/destDir";
    final String src = srcPath + "/file1";
    final String dest = destPath + "/file2";
    dfs.mkdirs(new Path(dfs.getUri() + srcPath));
    dfs.mkdirs(new Path(dfs.getUri() + destPath));
    // write to DISK
    final FSDataOutputStream out1 = dfs.create(new Path(dfs.getUri() + src));
    out1.writeChars("testCopy1");
    out1.close();
    final FSDataOutputStream out2 = dfs.create(new Path(dfs.getUri() + dest));
    out2.writeChars("testCopy2");
    out2.close();
    verifyRenameFailed(src, dest);
}

16 Source : TestSnapshotManifest.java
with Apache License 2.0
from fengchen8086

private void writeDataManifest(final SnapshotDataManifest manifest) throws IOException {
    FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME));
    try {
        manifest.writeTo(stream);
    } finally {
        stream.close();
    }
}

16 Source : TestHFile.java
with Apache License 2.0
from fengchen8086

/**
 * Create 0-length hfile and show that it fails
 */
public void testCorrupt0LengthHFile() throws IOException {
    if (cacheConf == null)
        cacheConf = new CacheConfig(conf);
    Path f = new Path(ROOT_DIR, getName());
    FSDataOutputStream fsos = fs.create(f);
    fsos.close();
    try {
        Reader r = HFile.createReader(fs, f, cacheConf, conf);
    } catch (CorruptHFileException che) {
        // Expected failure
        return;
    }
    fail("Should have thrown exception");
}

See More Examples