Package org.apache.pulsar.io.hdfs2

Class AbstractHdfsConnector

java.lang.Object

org.apache.pulsar.io.hdfs2.AbstractHdfsConnector

Direct Known Subclasses:

HdfsAbstractSink

public abstract class AbstractHdfsConnector
extends java.lang.Object

A Simple abstract class for HDFS connectors. Provides methods for connecting to HDFS

Field Summary

Fields

Modifier and Type	Field	Description
protected org.apache.hadoop.io.compress.CompressionCodecFactory	compressionCodecFactory
protected AbstractHdfsConfig	connectorConfig
protected java.util.concurrent.atomic.AtomicReference<HdfsResources>	hdfsResources

Constructor Summary

Constructors

Constructor	Description
AbstractHdfsConnector()

Method Summary

Modifier and Type	Method	Description
protected void	checkHdfsUriForTimeout(org.apache.hadoop.conf.Configuration config)
protected org.apache.hadoop.io.compress.CompressionCodec	getCompressionCodec()
protected org.apache.hadoop.io.compress.CompressionCodecFactory	getCompressionCodecFactory()
protected org.apache.hadoop.conf.Configuration	getConfiguration()
protected java.lang.String	getEncoding()
protected org.apache.hadoop.fs.FileSystem	getFileSystem()
protected org.apache.hadoop.fs.FileSystem	getFileSystem(org.apache.hadoop.conf.Configuration config)	This exists in order to allow unit tests to override it so that they don't take several minutes waiting for UDP packets to be received.
protected org.apache.hadoop.fs.FileSystem	getFileSystemAsUser(org.apache.hadoop.conf.Configuration config, org.apache.hadoop.security.UserGroupInformation ugi)
protected org.apache.hadoop.security.UserGroupInformation	getUserGroupInformation()
protected HdfsResources	resetHDFSResources(HdfsSinkConfig hdfsSinkConfig)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

hdfsResources

protected final java.util.concurrent.atomic.AtomicReference<HdfsResources> hdfsResources

connectorConfig

protected AbstractHdfsConfig connectorConfig

compressionCodecFactory

protected org.apache.hadoop.io.compress.CompressionCodecFactory compressionCodecFactory

Constructor Detail

AbstractHdfsConnector

public AbstractHdfsConnector()

Method Detail

resetHDFSResources

protected HdfsResources resetHDFSResources(HdfsSinkConfig hdfsSinkConfig)
                                    throws java.io.IOException

Throws:

java.io.IOException

checkHdfsUriForTimeout

protected void checkHdfsUriForTimeout(org.apache.hadoop.conf.Configuration config)
                               throws java.io.IOException

Throws:

java.io.IOException

getFileSystem

protected org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.conf.Configuration config)
                                                 throws java.io.IOException

This exists in order to allow unit tests to override it so that they don't take several minutes waiting for UDP packets to be received.

Parameters:

config - the configuration to use

Returns:

the FileSystem that is created for the given Configuration

Throws:

java.io.IOException - if unable to create the FileSystem

getFileSystemAsUser

protected org.apache.hadoop.fs.FileSystem getFileSystemAsUser(org.apache.hadoop.conf.Configuration config,
                                                              org.apache.hadoop.security.UserGroupInformation ugi)
                                                       throws java.io.IOException

Throws:

java.io.IOException

getConfiguration

protected org.apache.hadoop.conf.Configuration getConfiguration()

getFileSystem

protected org.apache.hadoop.fs.FileSystem getFileSystem()

getUserGroupInformation

protected org.apache.hadoop.security.UserGroupInformation getUserGroupInformation()

getEncoding

protected java.lang.String getEncoding()

getCompressionCodec

protected org.apache.hadoop.io.compress.CompressionCodec getCompressionCodec()

getCompressionCodecFactory

protected org.apache.hadoop.io.compress.CompressionCodecFactory getCompressionCodecFactory()