cascading.flow.hadoop.planner

Class HadoopFlowStepJob

java.lang.Object

cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

cascading.flow.hadoop.planner.HadoopFlowStepJob

All Implemented Interfaces:

java.util.concurrent.Callable<java.lang.Throwable>

public class HadoopFlowStepJob
extends cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.mapred.JobClient	jobClient Field jobClient
protected org.apache.hadoop.mapred.RunningJob	runningJob Field runningJob

Fields inherited from class cascading.flow.planner.FlowStepJob

blockForCompletedChildDetailDuration, flowStep, flowStepStats, jobConfiguration, pollingInterval, predecessors, statsStoreInterval, stepName, throwable

Constructor Summary

Constructors

Constructor and Description
HadoopFlowStepJob(cascading.management.state.ClientState clientState, cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf> flowStep, org.apache.hadoop.mapred.JobConf currentConf)

Method Summary

Modifier and Type	Method and Description
protected cascading.stats.FlowStepStats	createStepStats(cascading.management.state.ClientState clientState)
protected void	dumpDebugInfo()
static long	getJobPollingInterval(org.apache.hadoop.mapred.JobConf jobConf)
protected java.lang.Throwable	getThrowable()
protected void	internalBlockOnStop()
protected boolean	internalIsStartedRunning()
protected java.lang.String	internalJobId()
protected boolean	internalNonBlockingIsComplete()
protected boolean	internalNonBlockingIsSuccessful()
protected void	internalNonBlockingStart()
protected org.apache.hadoop.mapred.RunningJob	internalNonBlockingSubmit()
protected boolean	isRemoteExecution()
boolean	isSuccessful()
static void	reportLocalError(java.lang.Throwable throwable) Internal method to report errors that happen on hadoop local mode.
protected void	updateNodeStatus(cascading.stats.FlowNodeStats flowNodeStats)

Methods inherited from class cascading.flow.planner.FlowStepJob

blockOnJob, blockOnPredecessors, blockTillCompleteOrStopped, call, finalizeNodeSliceCapture, getConfig, getStepStats, internalCleanup, isCallableStarted, isSkipFlowStep, isStarted, setPredecessors, sleepForPollingInterval, start, stop

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

jobClient

protected org.apache.hadoop.mapred.JobClient jobClient

Field jobClient

runningJob

protected org.apache.hadoop.mapred.RunningJob runningJob

Field runningJob

Constructor Detail

HadoopFlowStepJob

public HadoopFlowStepJob(cascading.management.state.ClientState clientState,
                         cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf> flowStep,
                         org.apache.hadoop.mapred.JobConf currentConf)

Method Detail

getJobPollingInterval

public static long getJobPollingInterval(org.apache.hadoop.mapred.JobConf jobConf)

createStepStats

protected cascading.stats.FlowStepStats createStepStats(cascading.management.state.ClientState clientState)

Specified by:

createStepStats in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

internalBlockOnStop

protected void internalBlockOnStop()
                            throws java.io.IOException

Specified by:

internalBlockOnStop in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

Throws:

java.io.IOException

internalNonBlockingStart

protected void internalNonBlockingStart()
                                 throws java.io.IOException

Specified by:

internalNonBlockingStart in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

Throws:

java.io.IOException

internalNonBlockingSubmit

protected org.apache.hadoop.mapred.RunningJob internalNonBlockingSubmit()
                                                                 throws java.io.IOException

Throws:

java.io.IOException

updateNodeStatus

protected void updateNodeStatus(cascading.stats.FlowNodeStats flowNodeStats)

Specified by:

updateNodeStatus in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

isSuccessful

public boolean isSuccessful()

Overrides:

isSuccessful in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

internalNonBlockingIsSuccessful

protected boolean internalNonBlockingIsSuccessful()
                                           throws java.io.IOException

Specified by:

internalNonBlockingIsSuccessful in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

Throws:

java.io.IOException

isRemoteExecution

protected boolean isRemoteExecution()

Specified by:

isRemoteExecution in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

getThrowable

protected java.lang.Throwable getThrowable()

Specified by:

getThrowable in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

internalJobId

protected java.lang.String internalJobId()

Specified by:

internalJobId in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

internalNonBlockingIsComplete

protected boolean internalNonBlockingIsComplete()
                                         throws java.io.IOException

Specified by:

internalNonBlockingIsComplete in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

Throws:

java.io.IOException

dumpDebugInfo

protected void dumpDebugInfo()

Specified by:

dumpDebugInfo in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

internalIsStartedRunning

protected boolean internalIsStartedRunning()

Specified by:

internalIsStartedRunning in class cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf>

reportLocalError

public static void reportLocalError(java.lang.Throwable throwable)

Internal method to report errors that happen on hadoop local mode. Hadoops local JobRunner does not give access to TaskReports, but we want to be able to capture the exception and not just print it to stderr. FlowMapper and FlowReducer use this method.

Parameters:

throwable - the throwable to be reported.