Here are the examples of the java api org.apache.hadoop.fs.FSDataOutputStream.close() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
516 Examples
19
Source : BatchGroup.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
public void closeOutputStream() throws IOException {
if (outputStream != null) {
outputStream.close();
}
}
19
Source : GenericSourceSink.java
with Apache License 2.0
from snuspl
with Apache License 2.0
from snuspl
/**
* finish bundle.
* @param c context
* @throws IOException output stream exception
*/
@FinishBundle
public void finishBundle(final FinishBundleContext c) throws IOException {
outputStream.close();
}
19
Source : FileSystemNodeLabelsStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void ensureCloseEditlogFile() throws IOException {
editlogOs.close();
}
19
Source : FSPartFileCollection.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
private void turnover() throws IOException {
partOutStr.close();
partOutStr = null;
writeIndex();
syncRequested = false;
}
19
Source : VectorizedHashAggDiskPartition.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
/**
* Once {@link VectorizedHashAggOperator} has consumed all data, it flushes
* the in-memory data (if any) for all spilled parreplacedions and that's when we
* finally close the output stream for parreplacedion's spill file since that is the
* last moment we will spill anything for that parreplacedion as later on the parreplacedion
* will transition to read-only (pure disk based) mode.
* @throws Exception
*/
public void closeSpillStream() throws Exception {
outputStream.close();
outputStream = null;
}
19
Source : GenericSourceSink.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Teardown.
*
* @throws IOException output stream exception
*/
@Teardown
public void tearDown() throws IOException {
outputStream.close();
}
19
Source : HoodieLogFormatWriter.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void closeStream() throws IOException {
if (output != null) {
flush();
output.close();
output = null;
closed = true;
}
}
18
Source : JsonRecordWriter.java
with Apache License 2.0
from zpochen
with Apache License 2.0
from zpochen
@Override
public void cleanup() throws IOException {
gen.flush();
stream.close();
}
18
Source : HDFSSequenceFile.java
with MIT License
from TranswarpCN
with MIT License
from TranswarpCN
@Override
public void close() throws IOException {
writer.close();
outStream.close();
unregisterCurrentStream();
}
18
Source : TestTFileUnsortedByteArrays.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void closeOutput() throws IOException {
if (writer != null) {
writer.close();
writer = null;
out.close();
out = null;
}
}
18
Source : TestTFileStreams.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void closeOutput() throws IOException {
if (writer != null) {
writer.close();
writer = null;
}
if (out != null) {
out.close();
out = null;
}
}
18
Source : HadoopRecoverableFsDataOutputStream.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public void close() throws IOException {
out.close();
}
18
Source : StreamWriterBase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
@Override
public void close() throws IOException {
if (outStream != null) {
flush();
outStream.close();
outStream = null;
}
}
18
Source : FSJsonLineFile.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
@Override
public void close() throws IOException {
os.close();
}
18
Source : AbstractHFileWriter.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
/**
* Sets remaining trailer fields, writes the trailer to disk, and optionally
* closes the output stream.
*/
protected void finishClose(FixedFileTrailer trailer) throws IOException {
trailer.setMetaIndexCount(metaNames.size());
trailer.setTotalUncompressedBytes(totalUncompressedBytes + trailer.getTrailerSize());
trailer.setEntryCount(entryCount);
trailer.setCompressionCodec(hFileContext.getCompression());
trailer.serialize(outputStream);
if (closeOutputStream) {
outputStream.close();
outputStream = null;
}
}
18
Source : DirtyDataManager.java
with Apache License 2.0
from DTStack
with Apache License 2.0
from DTStack
public void close() {
if (stream != null) {
try {
stream.flush();
stream.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
18
Source : FSDataOutputStreamWrapper.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
@Override
public void close() throws IOException {
try {
underlyingOS.close();
} catch (FSError e) {
throw propagateFSError(e);
}
}
18
Source : FilterFSOutputStream.java
with Apache License 2.0
from dremio
with Apache License 2.0
from dremio
/**
* @throws IOException
* @see java.io.OutputStream#close()
*/
@Override
public void close() throws IOException {
out.close();
}
18
Source : ArchiveHdfsLogWriter.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void rollWriter() throws IOException {
if (lastRollRecordId != currentRecordId) {
// close old file
os.close();
hdfs.rename(currentPath, new Path(LogServiceUtils.getRolledPathForArchiveWriter(currentPath, currentRecordId)));
lastRollRecordId = currentRecordId;
// create new file
os = hdfs.create(currentPath, true);
}
}
18
Source : ArchiveHdfsLogWriter.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void close() throws IOException {
os.close();
if (lastRollRecordId != currentRecordId) {
hdfs.rename(currentPath, new Path(currentPath + "_recordId_" + currentRecordId));
}
}
18
Source : HDFSOutput.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void close() throws IOException {
fsDataOutputStream.close();
}
17
Source : TestRubixCaching.java
with Apache License 2.0
from trinodb
with Apache License 2.0
from trinodb
private void writeFile(FSDataOutputStream outputStream, byte[] content) throws IOException {
try {
outputStream.write(content);
} finally {
outputStream.close();
}
}
17
Source : GenericSourceSink.java
with Apache License 2.0
from snuspl
with Apache License 2.0
from snuspl
/**
* process element.
* @param c context {@link ProcessContext}
* @throws Exception exception.
*/
@ProcessElement
public void processElement(final ProcessContext c) throws Exception {
try {
outputStream.writeBytes(c.element() + "\n");
} catch (Exception e) {
outputStream.close();
fileSystem.delete(fileName, true);
throw new RuntimeException(e);
}
}
17
Source : NNBenchWithoutMR.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Create and write to a given number of files. Repeat each remote
* operation until is suceeds (does not throw an exception).
*
* @return the number of exceptions caught
*/
static int createWrite() {
int totalExceptions = 0;
FSDataOutputStream out = null;
boolean success;
for (int index = 0; index < numFiles; index++) {
int singleFileExceptions = 0;
do {
// create file until is succeeds or max exceptions reached
try {
out = fileSys.create(new Path(taskDir, "" + index), false, 512, (short) 1, bytesPerBlock);
success = true;
} catch (IOException ioe) {
success = false;
totalExceptions++;
handleException("creating file #" + index, ioe, ++singleFileExceptions);
}
} while (!success);
long toBeWritten = bytesPerFile;
while (toBeWritten > 0) {
int nbytes = (int) Math.min(buffer.length, toBeWritten);
toBeWritten -= nbytes;
try {
// only try once
out.write(buffer, 0, nbytes);
} catch (IOException ioe) {
totalExceptions++;
handleException("writing to file #" + index, ioe, ++singleFileExceptions);
}
}
do {
// close file until is succeeds
try {
out.close();
success = true;
} catch (IOException ioe) {
success = false;
totalExceptions++;
handleException("closing file #" + index, ioe, ++singleFileExceptions);
}
} while (!success);
}
return totalExceptions;
}
17
Source : EventWriter.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
void close() throws IOException {
try {
encoder.flush();
out.close();
out = null;
} finally {
IOUtils.cleanup(LOG, out);
}
}
17
Source : TestFileAppend3.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
// Do small appends.
void doSmallAppends(Path file, DistributedFileSystem fs, int iterations) throws IOException {
for (int i = 0; i < iterations; i++) {
FSDataOutputStream stm;
try {
stm = fs.append(file);
} catch (IOException e) {
// If another thread is already appending, skip this time.
continue;
}
// Failure in write or close will be terminal.
AppendTestUtil.write(stm, 0, 123);
stm.close();
}
}
17
Source : TestTFileByteArrays.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testFailureOpenEmptyFile() throws IOException {
if (skip)
return;
closeOutput();
// create an absolutely empty file
path = new Path(fs.getWorkingDirectory(), outputFile);
out = fs.create(path);
out.close();
try {
new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
replacedert.fail("Error on handling empty files.");
} catch (EOFException e) {
// noop, expecting exceptions
}
}
17
Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testCorruptedChecksum() throws Exception {
Path testPath = new Path(TEST_ROOT_DIR, "testCorruptChecksum");
Path checksumPath = localFs.getChecksumFile(testPath);
// write a file to generate checksum
FSDataOutputStream out = localFs.create(testPath, true);
out.write("testing 1 2 3".getBytes());
out.close();
replacedertTrue(localFs.exists(checksumPath));
FileStatus stat = localFs.getFileStatus(checksumPath);
// alter file directly so checksum is invalid
out = localFs.getRawFileSystem().create(testPath, true);
out.write("testing stale checksum".getBytes());
out.close();
replacedertTrue(localFs.exists(checksumPath));
// checksum didn't change on disk
replacedertEquals(stat, localFs.getFileStatus(checksumPath));
Exception e = null;
try {
localFs.setVerifyChecksum(true);
readFile(localFs, testPath, 1024);
} catch (ChecksumException ce) {
e = ce;
} finally {
replacedertNotNull("got checksum error", e);
}
localFs.setVerifyChecksum(false);
String str = readFile(localFs, testPath, 1024);
replacedertEquals("testing stale checksum", str);
}
17
Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testVerifyChecksum() throws Exception {
Path testPath = new Path(TEST_ROOT_DIR, "testPath");
Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
FSDataOutputStream fout = localFs.create(testPath);
fout.write("testing".getBytes());
fout.close();
fout = localFs.create(testPath11);
fout.write("testing you".getBytes());
fout.close();
// Exercise some boundary cases - a divisor of the chunk size
// the chunk size, 2x chunk size, and +/-1 around these.
readFile(localFs, testPath, 128);
readFile(localFs, testPath, 511);
readFile(localFs, testPath, 512);
readFile(localFs, testPath, 513);
readFile(localFs, testPath, 1023);
readFile(localFs, testPath, 1024);
readFile(localFs, testPath, 1025);
localFs.delete(localFs.getChecksumFile(testPath), true);
replacedertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
// copying the wrong checksum file
FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs, localFs.getChecksumFile(testPath), false, true, localFs.getConf());
replacedertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
boolean errorRead = false;
try {
readFile(localFs, testPath, 1024);
} catch (ChecksumException ie) {
errorRead = true;
}
replacedertTrue("error reading", errorRead);
// now setting verify false, the read should succeed
localFs.setVerifyChecksum(false);
String str = readFile(localFs, testPath, 1024).toString();
replacedertTrue("read", "testing".equals(str));
}
17
Source : TestChecksumFileSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testMultiChunkFile() throws Exception {
Path testPath = new Path(TEST_ROOT_DIR, "testMultiChunk");
FSDataOutputStream fout = localFs.create(testPath);
for (int i = 0; i < 1000; i++) {
fout.write(("testing" + i).getBytes());
}
fout.close();
// Exercise some boundary cases - a divisor of the chunk size
// the chunk size, 2x chunk size, and +/-1 around these.
readFile(localFs, testPath, 128);
readFile(localFs, testPath, 511);
readFile(localFs, testPath, 512);
readFile(localFs, testPath, 513);
readFile(localFs, testPath, 1023);
readFile(localFs, testPath, 1024);
readFile(localFs, testPath, 1025);
}
17
Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testAppendMissingTarget() throws Throwable {
try {
FSDataOutputStream out = getFileSystem().append(target);
// got here: trouble
out.close();
fail("expected a failure");
} catch (Exception e) {
// expected
handleExpectedException(e);
}
}
17
Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testAppendNonexistentFile() throws Throwable {
try {
FSDataOutputStream out = getFileSystem().append(target);
// got here: trouble
out.close();
fail("expected a failure");
} catch (Exception e) {
// expected
handleExpectedException(e);
}
}
17
Source : HiveProducer.java
with Apache License 2.0
from Kyligence
with Apache License 2.0
from Kyligence
private void closeFout() {
if (fout != null) {
try {
fout.close();
} catch (Exception e) {
logger.error("Close the path: " + curParreplacedionContentPath + " failed", e);
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem hdfs = (DistributedFileSystem) fs;
try {
boolean recovered = hdfs.recoverLease(curParreplacedionContentPath);
} catch (Exception e1) {
logger.error("Recover lease for path: " + curParreplacedionContentPath + " failed", e1);
}
}
}
}
fout = null;
}
17
Source : DataFile.java
with Apache License 2.0
from didi
with Apache License 2.0
from didi
public boolean close() {
boolean flag = false;
closeInvokedCount++;
try {
flush();
stream.close();
flag = true;
closed = true;
} catch (Exception e) {
LOGGER.error("Close file error, dir:" + dir + ", file:" + fileName, e);
}
if (flag) {
dir.deleteDataFile(pid);
LOGGER.info("Close file success,file:" + getPath());
} else {
LOGGER.info("Close file failed, then add CloseFailStream,file:" + getPath());
dataFileManager.addCloseFailedStream(this);
}
return flag;
}
17
Source : DataFile.java
with Apache License 2.0
from didi
with Apache License 2.0
from didi
public boolean directClose() {
boolean flag = false;
closeInvokedCount++;
try {
flush();
stream.close();
flag = true;
closed = true;
} catch (Exception e) {
LOGGER.error("Close file error, dir:" + dir + ", file:" + fileName, e);
}
if (flag) {
LOGGER.info("Close file success,file:" + getPath());
} else {
LOGGER.info("Close file failed, then add CloseFailStream,file:" + getPath());
dataFileManager.addCloseFailedStream(this);
}
dir.deleteDataFile(pid);
return flag;
}
17
Source : HdfsTransactionLog.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void doCloseOutput() throws IOException {
synchronized (this) {
if (fos == null)
return;
if (debug) {
log.debug("Closing output for {}", tlogFile);
}
fos.flushBuffer();
finalLogSize = fos.size();
fos = null;
}
tlogOutStream.hflush();
tlogOutStream.close();
tlogOutStream = null;
}
17
Source : GenericSourceSink.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* process element.
*
* @param c context {@link ProcessContext}
* @throws Exception exception.
*/
@ProcessElement
public void processElement(final ProcessContext c) throws DataSourceException, IOException {
try {
outputStream.writeBytes(c.element() + "\n");
} catch (Exception e) {
outputStream.close();
fileSystem.delete(fileName, true);
throw new DataSourceException(new Exception("Processing data from source failed " + e));
}
}
16
Source : FileSystemNodeLabelsStore.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public void close() throws IOException {
try {
fs.close();
editlogOs.close();
} catch (IOException e) {
LOG.warn("Exception happened whiling shutting down,", e);
}
}
16
Source : TestSwiftFileSystemExtendedContract.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testCreateFile() throws Exception {
final Path f = new Path("/test/testCreateFile");
final FSDataOutputStream fsDataOutputStream = fs.create(f);
fsDataOutputStream.close();
replacedertExists("created file", f);
}
16
Source : SwiftFileSystemBaseTest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Create and then close a file
* @param path path to create
* @throws IOException on a failure
*/
protected void createEmptyFile(Path path) throws IOException {
FSDataOutputStream out = fs.create(path);
out.close();
}
16
Source : TestUserResolve.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Creates users file with the content as the String usersFileContent.
* @param usersFilePath the path to the file that is to be created
* @param usersFileContent Content of users file
* @throws IOException
*/
private static void writeUserList(Path usersFilePath, String usersFileContent) throws IOException {
FSDataOutputStream out = null;
try {
out = fs.create(usersFilePath, true);
out.writeBytes(usersFileContent);
} finally {
if (out != null) {
out.close();
}
}
}
16
Source : TestFileConcurrentReader.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Test that that writes to an incomplete block are available to a reader
*/
@Test(timeout = 30000)
public void testUnfinishedBlockRead() throws IOException {
// create a new file in the root, write data, do no close
Path file1 = new Path("/unfinished-block");
FSDataOutputStream stm = TestFileCreation.createFile(fileSystem, file1, 1);
// write partial block and sync
int partialBlockSize = blockSize / 2;
writeFileAndSync(stm, partialBlockSize);
// Make sure a client can read it before it is closed
checkCanRead(fileSystem, file1, partialBlockSize);
stm.close();
}
16
Source : AbstractContractAppendTest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testAppendToExistingFile() throws Throwable {
byte[] original = dataset(8192, 'A', 'Z');
byte[] appended = dataset(8192, '0', '9');
createFile(getFileSystem(), target, false, original);
FSDataOutputStream outputStream = getFileSystem().append(target);
outputStream.write(appended);
outputStream.close();
byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), target, original.length + appended.length);
ContractTestUtils.validateFileContent(bytes, new byte[][] { original, appended });
}
16
Source : TestDecommission.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private void writeConfigFile(Path name, List<String> nodes) throws IOException {
// delete if it already exists
if (localFileSys.exists(name)) {
localFileSys.delete(name, true);
}
FSDataOutputStream stm = localFileSys.create(name);
if (nodes != null) {
for (Iterator<String> it = nodes.iterator(); it.hasNext(); ) {
String node = it.next();
stm.writeBytes(node);
stm.writeBytes("\n");
}
}
stm.close();
}
16
Source : ContinuousFileProcessingITCase.java
with Apache License 2.0
from ljygz
with Apache License 2.0
from ljygz
/**
* Create a file and fill it with content.
*/
private Tuple2<org.apache.hadoop.fs.Path, String> fillWithData(String base, String fileName, int fileIdx, String sampleLine) throws IOException, InterruptedException {
replacedert (hdfs != null);
org.apache.hadoop.fs.Path tmp = new org.apache.hadoop.fs.Path(base + "/." + fileName + fileIdx);
FSDataOutputStream stream = hdfs.create(tmp);
StringBuilder str = new StringBuilder();
for (int i = 0; i < LINES_PER_FILE; i++) {
String line = fileIdx + ": " + sampleLine + " " + i + "\n";
str.append(line);
stream.write(line.getBytes(ConfigConstants.DEFAULT_CHARSET));
}
stream.close();
return new Tuple2<>(tmp, str.toString());
}
16
Source : FSPartFileCollection.java
with Apache License 2.0
from lealone
with Apache License 2.0
from lealone
public void teardown() {
logger.info("Closing hdfs part collection.");
try {
if (metaOs != null) {
metaOs.close();
}
if (partOutStr != null) {
logger.debug("Closing part file");
partOutStr.close();
if (indexOutStr != null) {
writeIndex();
}
}
if (indexOutStr != null) {
writeIndexEnd();
indexOutStr.close();
}
fs.close();
} catch (IOException ex) {
logger.error(ex.toString());
}
}
16
Source : TestCmdlet.java
with Apache License 2.0
from Intel-bigdata
with Apache License 2.0
from Intel-bigdata
private void generateTestFiles() throws IOException {
// New dir
Path dir = new Path("/testMoveFile");
dfs.mkdirs(dir);
// Move to SSD
dfs.setStoragePolicy(dir, "HOT");
final FSDataOutputStream out1 = dfs.create(new Path("/testMoveFile/file1"), true, 1024);
out1.writeChars("/testMoveFile/file1");
out1.close();
// Move to Archive
final FSDataOutputStream out2 = dfs.create(new Path("/testMoveFile/file2"), true, 1024);
out2.writeChars("/testMoveFile/file2");
out2.close();
// Move to CacheObject
Path dir3 = new Path("/testCacheFile");
dfs.mkdirs(dir3);
}
16
Source : TestRenameFileAction.java
with Apache License 2.0
from Intel-bigdata
with Apache License 2.0
from Intel-bigdata
@Test
public void testDestExistingFileRename() throws Exception {
final String srcPath = "/testRename";
final String destPath = "/destDir";
final String src = srcPath + "/file1";
final String dest = destPath + "/file2";
dfs.mkdirs(new Path(dfs.getUri() + srcPath));
dfs.mkdirs(new Path(dfs.getUri() + destPath));
// write to DISK
final FSDataOutputStream out1 = dfs.create(new Path(dfs.getUri() + src));
out1.writeChars("testCopy1");
out1.close();
final FSDataOutputStream out2 = dfs.create(new Path(dfs.getUri() + dest));
out2.writeChars("testCopy2");
out2.close();
verifyRenameFailed(src, dest);
}
16
Source : TestSnapshotManifest.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
private void writeDataManifest(final SnapshotDataManifest manifest) throws IOException {
FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME));
try {
manifest.writeTo(stream);
} finally {
stream.close();
}
}
16
Source : TestHFile.java
with Apache License 2.0
from fengchen8086
with Apache License 2.0
from fengchen8086
/**
* Create 0-length hfile and show that it fails
*/
public void testCorrupt0LengthHFile() throws IOException {
if (cacheConf == null)
cacheConf = new CacheConfig(conf);
Path f = new Path(ROOT_DIR, getName());
FSDataOutputStream fsos = fs.create(f);
fsos.close();
try {
Reader r = HFile.createReader(fs, f, cacheConf, conf);
} catch (CorruptHFileException che) {
// Expected failure
return;
}
fail("Should have thrown exception");
}
See More Examples