cascading.flow.tez

Class Hadoop2TezFlowStep

java.lang.Object

cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

cascading.flow.tez.Hadoop2TezFlowStep

All Implemented Interfaces:

cascading.flow.FlowStep<org.apache.tez.dag.api.TezConfiguration>, cascading.flow.planner.process.ProcessModel, cascading.util.ProcessLogger, java.io.Serializable

public class Hadoop2TezFlowStep
extends cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

See Also:

Serialized Form

Field Summary

Fields inherited from class cascading.flow.planner.BaseFlowStep

elementGraph, flowNodeGraph, flowStepStats, sinks, sources, tempSink

Fields inherited from interface cascading.flow.FlowStep

CASCADING_FLOW_STEP_ID

Fields inherited from interface cascading.util.ProcessLogger

NULL

Constructor Summary

Constructors

Constructor and Description
Hadoop2TezFlowStep(cascading.flow.planner.graph.ElementGraph elementGraph, cascading.flow.planner.process.FlowNodeGraph flowNodeGraph)

Method Summary

Modifier and Type	Method and Description
void	clean(org.apache.tez.dag.api.TezConfiguration config)
protected cascading.flow.planner.FlowStepJob	createFlowStepJob(cascading.management.state.ClientState clientState, cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess, org.apache.tez.dag.api.TezConfiguration initializedStepConfig)
org.apache.tez.dag.api.TezConfiguration	createInitializedConfig(cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess, org.apache.tez.dag.api.TezConfiguration parentConfig)
org.apache.tez.dag.api.Vertex	createVertex(cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess, org.apache.tez.dag.api.TezConfiguration initializedConfig, cascading.flow.FlowNode flowNode)
java.util.Map<java.lang.String,org.apache.hadoop.yarn.api.records.LocalResource>	getAllLocalResources()
java.util.Map<java.lang.Object,java.lang.Object>	getConfigAsProperties()
protected java.lang.String	getOutputPath(org.apache.hadoop.conf.Configuration sinkConf)
protected int	getParallelism(cascading.flow.FlowNode flowNode, org.apache.hadoop.mapred.JobConf conf)
protected java.util.Map<cascading.flow.FlowElement,org.apache.hadoop.conf.Configuration>	initFromSinks(cascading.flow.FlowNode flowNode, cascading.flow.FlowProcess<? extends org.apache.hadoop.conf.Configuration> flowProcess, org.apache.hadoop.conf.Configuration conf)
protected java.util.Map<cascading.flow.FlowElement,org.apache.hadoop.conf.Configuration>	initFromSources(cascading.flow.FlowNode flowNode, cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess, org.apache.hadoop.conf.Configuration conf, java.util.Map<java.lang.String,org.apache.hadoop.yarn.api.records.LocalResource> taskLocalResources)
protected void	initFromTraps(cascading.flow.FlowNode flowNode, cascading.flow.FlowProcess<? extends org.apache.hadoop.conf.Configuration> flowProcess, org.apache.hadoop.conf.Configuration conf)
protected boolean	isFileOutputFormat(java.lang.Class outputFormatClass)
void	syncArtifacts()

Methods inherited from class cascading.flow.planner.BaseFlowStep

addAllGroups, addGroup, addGroups, addListener, addProcessAnnotation, addProcessAnnotation, addSink, addSinks, addSource, addSources, clean, commitSinks, configure, containsPipeNamed, createClientState, equals, findAllSerializableTypes, fireOnCompleted, fireOnRunning, fireOnStarting, fireOnStopping, fireOnThrowable, getAllAccumulatedSources, getAllOperations, getAllStreamedSources, getAnnotations, getConfig, getCreateFlowStepJob, getElementGraph, getFieldDeclaredSerializations, getFlow, getFlowID, getFlowName, getFlowNodeGraph, getFlowStepDescriptor, getFlowStepJob, getFlowStepStats, getGroup, getGroups, getID, getName, getNextFlowElement, getNextScope, getNodeDisplayName, getNumFlowNodes, getOrdinal, getPreviousScopes, getProcessAnnotations, getSink, getSinkElements, getSinkName, getSinkTaps, getSinkWith, getSourceElements, getSourceName, getSourceTaps, getSourceWith, getStepDisplayName, getStepDisplayName, getSubmitPriority, getTrap, getTrapMap, getTraps, hashCode, hasListeners, initConfFromNodeConfigDef, initConfFromStepConfigDef, isDebugEnabled, isInfoEnabled, logDebug, logError, logError, logInfo, logWarn, logWarn, logWarn, prepareResources, removeListener, rollbackSinks, setConfig, setFlow, setFlowName, setFlowStepDescriptor, setFlowStepStats, setName, setOrdinal, setSubmitPriority, toString

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Constructor Detail

Hadoop2TezFlowStep

public Hadoop2TezFlowStep(cascading.flow.planner.graph.ElementGraph elementGraph,
                          cascading.flow.planner.process.FlowNodeGraph flowNodeGraph)

Method Detail

getConfigAsProperties

public java.util.Map<java.lang.Object,java.lang.Object> getConfigAsProperties()

Specified by:

getConfigAsProperties in interface cascading.flow.FlowStep<org.apache.tez.dag.api.TezConfiguration>

Overrides:

getConfigAsProperties in class cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

createInitializedConfig

public org.apache.tez.dag.api.TezConfiguration createInitializedConfig(cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess,
                                                                       org.apache.tez.dag.api.TezConfiguration parentConfig)

Specified by:

createInitializedConfig in class cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

createFlowStepJob

protected cascading.flow.planner.FlowStepJob createFlowStepJob(cascading.management.state.ClientState clientState,
                                                               cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess,
                                                               org.apache.tez.dag.api.TezConfiguration initializedStepConfig)

Specified by:

createFlowStepJob in class cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

createVertex

public org.apache.tez.dag.api.Vertex createVertex(cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess,
                                                  org.apache.tez.dag.api.TezConfiguration initializedConfig,
                                                  cascading.flow.FlowNode flowNode)

getOutputPath

protected java.lang.String getOutputPath(org.apache.hadoop.conf.Configuration sinkConf)

isFileOutputFormat

protected boolean isFileOutputFormat(java.lang.Class outputFormatClass)

getParallelism

protected int getParallelism(cascading.flow.FlowNode flowNode,
                             org.apache.hadoop.mapred.JobConf conf)

initFromSources

protected java.util.Map<cascading.flow.FlowElement,org.apache.hadoop.conf.Configuration> initFromSources(cascading.flow.FlowNode flowNode,
                                                                                                         cascading.flow.FlowProcess<org.apache.tez.dag.api.TezConfiguration> flowProcess,
                                                                                                         org.apache.hadoop.conf.Configuration conf,
                                                                                                         java.util.Map<java.lang.String,org.apache.hadoop.yarn.api.records.LocalResource> taskLocalResources)

initFromSinks

protected java.util.Map<cascading.flow.FlowElement,org.apache.hadoop.conf.Configuration> initFromSinks(cascading.flow.FlowNode flowNode,
                                                                                                       cascading.flow.FlowProcess<? extends org.apache.hadoop.conf.Configuration> flowProcess,
                                                                                                       org.apache.hadoop.conf.Configuration conf)

initFromTraps

protected void initFromTraps(cascading.flow.FlowNode flowNode,
                             cascading.flow.FlowProcess<? extends org.apache.hadoop.conf.Configuration> flowProcess,
                             org.apache.hadoop.conf.Configuration conf)

clean

public void clean(org.apache.tez.dag.api.TezConfiguration config)

Specified by:

clean in class cascading.flow.planner.BaseFlowStep<org.apache.tez.dag.api.TezConfiguration>

syncArtifacts

public void syncArtifacts()

getAllLocalResources

public java.util.Map<java.lang.String,org.apache.hadoop.yarn.api.records.LocalResource> getAllLocalResources()