SnappyStreamingContext

Instance Constructors

new SnappyStreamingContext(path: String, sparkContext: SparkContext)

Recreate a SnappyStreamingContext from a checkpoint file using an existing SparkContext.
Recreate a SnappyStreamingContext from a checkpoint file using an existing SparkContext.
path
Path to the directory that was specified as the checkpoint directory
sparkContext
Existing SparkContext
new SnappyStreamingContext(path: String)

Recreate a SnappyStreamingContext from a checkpoint file.
Recreate a SnappyStreamingContext from a checkpoint file.
path
Path to the directory that was specified as the checkpoint directory
new SnappyStreamingContext(path: String, hadoopConf: Configuration)

Recreate a SnappyStreamingContext from a checkpoint file.
Recreate a SnappyStreamingContext from a checkpoint file.
path
Path to the directory that was specified as the checkpoint directory
hadoopConf
Optional, configuration object if necessary for reading from HDFS compatible filesystems
new SnappyStreamingContext(conf: SparkConf, batchDuration: Duration)

Create a SnappyStreamingContext by providing the configuration necessary for a new SparkContext.
Create a SnappyStreamingContext by providing the configuration necessary for a new SparkContext.
conf
a org.apache.spark.SparkConf object specifying Spark parameters
batchDuration
the time interval at which streaming data will be divided into batches
new SnappyStreamingContext(sparkContext: SparkContext, batchDuration: Duration)

Create a SnappyStreamingContext using an existing SparkContext.
Create a SnappyStreamingContext using an existing SparkContext.
sparkContext
existing SparkContext
batchDuration
the time interval at which streaming data will be divided into batches
new SnappyStreamingContext(sc_: SparkContext, cp_: Checkpoint, batchDur_: Duration)

Attributes
protected[org.apache.spark]

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def actorStream[T](props: Props, name: String, storageLevel: StorageLevel, supervisorStrategy: SupervisorStrategy)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def addStreamingListener(streamingListener: StreamingListener): Unit

Definition Classes
StreamingContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
def awaitTermination(): Unit

Definition Classes
StreamingContext
def awaitTerminationOrTimeout(timeout: Long): Boolean

Definition Classes
StreamingContext
def binaryRecordsStream(directory: String, recordLength: Int): DStream[Array[Byte]]

Definition Classes
StreamingContext
def checkpoint(directory: String): Unit

Definition Classes
StreamingContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createSchemaDStream(rowStream: DStream[Row], schema: StructType): SchemaDStream
def createSchemaDStream[A <: Product](stream: DStream[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaDStream

Creates a SchemaDStream from an DStream of Product (e.
Creates a SchemaDStream from an DStream of Product (e.g. case classes).
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean, conf: Configuration)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
def fileStream[K, V, F <: InputFormat[K, V]](directory: String)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getSchemaDStream(tableName: String): SchemaDStream
def getState(): StreamingContextState

Definition Classes
StreamingContext
Annotations
@DeveloperApi()
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean, defaultRDD: RDD[T])(implicit arg0: ClassTag[T]): InputDStream[T]

Definition Classes
StreamingContext
def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean)(implicit arg0: ClassTag[T]): InputDStream[T]

Definition Classes
StreamingContext
def rawSocketStream[T](hostname: String, port: Int, storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def receiverStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def registerCQ(queryStr: String): SchemaDStream

Registers and executes given SQL query and returns SchemaDStream to consume the results
Registers and executes given SQL query and returns SchemaDStream to consume the results
queryStr
returns
def remember(duration: Duration): Unit

Definition Classes
StreamingContext
val snappyContext: SnappyContext
def socketStream[T](hostname: String, port: Int, converter: (InputStream) ⇒ Iterator[T], storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def socketTextStream(hostname: String, port: Int, storageLevel: StorageLevel): ReceiverInputDStream[String]

Definition Classes
StreamingContext
def sparkContext: SparkContext

Definition Classes
StreamingContext
def sql(sqlText: String): DataFrame
def start(): Unit

Start the execution of the streams.
Start the execution of the streams. Also registers population of AQP tables from stream tables if present.

Definition Classes
SnappyStreamingContext → StreamingContext
Exceptions thrown
IllegalStateException
if the StreamingContext is already stopped
def stop(stopSparkContext: Boolean, stopGracefully: Boolean): Unit

Definition Classes
SnappyStreamingContext → StreamingContext
def stop(stopSparkContext: Boolean): Unit

Definition Classes
StreamingContext
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def textFileStream(directory: String): DStream[String]

Definition Classes
StreamingContext
def toString(): String

Definition Classes
AnyRef → Any
def transform[T](dstreams: Seq[DStream[_]], transformFunc: (Seq[RDD[_]], Time) ⇒ RDD[T])(implicit arg0: ClassTag[T]): DStream[T]

Definition Classes
StreamingContext
def union[T](streams: Seq[DStream[T]])(implicit arg0: ClassTag[T]): DStream[T]

Definition Classes
StreamingContext
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def awaitTermination(timeout: Long): Unit

Definition Classes
StreamingContext
Annotations
@deprecated
Deprecated
(Since version 1.3.0) Use awaitTerminationOrTimeout(Long) instead
def networkStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
Annotations
@deprecated
Deprecated
(Since version 1.0.0) Use receiverStream

class SnappyStreamingContext extends StreamingContext with Serializable

Instance Constructors

new SnappyStreamingContext(path: String, sparkContext: SparkContext)

new SnappyStreamingContext(path: String)

new SnappyStreamingContext(path: String, hadoopConf: Configuration)

new SnappyStreamingContext(conf: SparkConf, batchDuration: Duration)

new SnappyStreamingContext(sparkContext: SparkContext, batchDuration: Duration)

new SnappyStreamingContext(sc_: SparkContext, cp_: Checkpoint, batchDur_: Duration)

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def actorStream[T](props: Props, name: String, storageLevel: StorageLevel, supervisorStrategy: SupervisorStrategy)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def addStreamingListener(streamingListener: StreamingListener): Unit

final def asInstanceOf[T0]: T0

def awaitTermination(): Unit

def awaitTerminationOrTimeout(timeout: Long): Boolean

def binaryRecordsStream(directory: String, recordLength: Int): DStream[Array[Byte]]

def checkpoint(directory: String): Unit

def clone(): AnyRef

def createSchemaDStream(rowStream: DStream[Row], schema: StructType): SchemaDStream

def createSchemaDStream[A <: Product](stream: DStream[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): SchemaDStream

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean, conf: Configuration)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

def fileStream[K, V, F <: InputFormat[K, V]](directory: String)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

def finalize(): Unit

final def getClass(): Class[_]

def getSchemaDStream(tableName: String): SchemaDStream

def getState(): StreamingContextState

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean, defaultRDD: RDD[T])(implicit arg0: ClassTag[T]): InputDStream[T]

def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean)(implicit arg0: ClassTag[T]): InputDStream[T]

def rawSocketStream[T](hostname: String, port: Int, storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def receiverStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def registerCQ(queryStr: String): SchemaDStream

def remember(duration: Duration): Unit

val snappyContext: SnappyContext

def socketStream[T](hostname: String, port: Int, converter: (InputStream) ⇒ Iterator[T], storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def socketTextStream(hostname: String, port: Int, storageLevel: StorageLevel): ReceiverInputDStream[String]

def sparkContext: SparkContext

def sql(sqlText: String): DataFrame

def start(): Unit

def stop(stopSparkContext: Boolean, stopGracefully: Boolean): Unit

def stop(stopSparkContext: Boolean): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def textFileStream(directory: String): DStream[String]

def toString(): String

def transform[T](dstreams: Seq[DStream[_]], transformFunc: (Seq[RDD[_]], Time) ⇒ RDD[T])(implicit arg0: ClassTag[T]): DStream[T]

def union[T](streams: Seq[DStream[T]])(implicit arg0: ClassTag[T]): DStream[T]

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def awaitTermination(timeout: Long): Unit

def networkStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Inherited from Serializable

Inherited from Serializable

Inherited from StreamingContext