Package org.apache.pulsar.io.hdfs2.sink

Class HdfsAbstractSink<K,V>

java.lang.Object

org.apache.pulsar.io.hdfs2.AbstractHdfsConnector

org.apache.pulsar.io.hdfs2.sink.HdfsAbstractSink<K,V>

All Implemented Interfaces:

java.lang.AutoCloseable, org.apache.pulsar.io.core.Sink<V>

Direct Known Subclasses:

HdfsAbstractSequenceFileSink, HdfsAbstractTextFileSink

public abstract class HdfsAbstractSink<K,V>
extends AbstractHdfsConnector
implements org.apache.pulsar.io.core.Sink<V>

A Simple abstract class for HDFS sink. Users need to implement extractKeyValue function to use this sink.

Field Summary

Fields

Modifier and Type	Field	Description
protected HdfsSinkConfig	hdfsSinkConfig
protected HdfsSyncThread<V>	syncThread
protected java.util.concurrent.BlockingQueue<org.apache.pulsar.functions.api.Record<V>>	unackedRecords

Fields inherited from class org.apache.pulsar.io.hdfs2.AbstractHdfsConnector

compressionCodecFactory, connectorConfig, hdfsResources

Constructor Summary

Constructors

Constructor	Description
HdfsAbstractSink()

Method Summary

Modifier and Type	Method	Description
void	close()
protected void	connectToHdfs()
protected abstract void	createWriter()
abstract org.apache.pulsar.io.core.KeyValue<K,V>	extractKeyValue(org.apache.pulsar.functions.api.Record<V> record)
protected org.apache.hadoop.fs.FSDataOutputStream	getHdfsStream()
protected org.apache.hadoop.fs.Path	getPath()
protected void	launchSyncThread()
void	open(java.util.Map<java.lang.String,java.lang.Object> config, org.apache.pulsar.io.core.SinkContext sinkContext)

Methods inherited from class org.apache.pulsar.io.hdfs2.AbstractHdfsConnector

checkHdfsUriForTimeout, getCompressionCodec, getCompressionCodecFactory, getConfiguration, getEncoding, getFileSystem, getFileSystem, getFileSystemAsUser, getUserGroupInformation, resetHDFSResources

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.pulsar.io.core.Sink

write

Field Detail

hdfsSinkConfig

protected HdfsSinkConfig hdfsSinkConfig

unackedRecords

protected java.util.concurrent.BlockingQueue<org.apache.pulsar.functions.api.Record<V>> unackedRecords

syncThread

protected HdfsSyncThread<V> syncThread

Constructor Detail

HdfsAbstractSink

public HdfsAbstractSink()

Method Detail

extractKeyValue

public abstract org.apache.pulsar.io.core.KeyValue<K,V> extractKeyValue(org.apache.pulsar.functions.api.Record<V> record)

createWriter

protected abstract void createWriter()
                              throws java.io.IOException

Throws:

java.io.IOException

open

public void open(java.util.Map<java.lang.String,java.lang.Object> config,
                 org.apache.pulsar.io.core.SinkContext sinkContext)
          throws java.lang.Exception

Specified by:

open in interface org.apache.pulsar.io.core.Sink<K>

Throws:

java.lang.Exception

close

public void close()
           throws java.lang.Exception

Specified by:

close in interface java.lang.AutoCloseable

Throws:

java.lang.Exception

connectToHdfs

protected final void connectToHdfs()
                            throws java.io.IOException

Throws:

java.io.IOException

getHdfsStream

protected org.apache.hadoop.fs.FSDataOutputStream getHdfsStream()
                                                         throws java.lang.IllegalArgumentException,
                                                                java.io.IOException

Throws:

java.lang.IllegalArgumentException

java.io.IOException

getPath

protected final org.apache.hadoop.fs.Path getPath()

launchSyncThread

protected final void launchSyncThread()
                               throws java.io.IOException

Throws:

java.io.IOException