cascading.flow.hadoop

Class BaseMapReduceFlow

java.lang.Object

cascading.flow.BaseFlow<org.apache.hadoop.mapred.JobConf>

cascading.flow.hadoop.HadoopFlow

cascading.flow.hadoop.BaseMapReduceFlow

All Implemented Interfaces:

cascading.flow.Flow<org.apache.hadoop.mapred.JobConf>, cascading.management.UnitOfWork<cascading.stats.FlowStats>, cascading.util.ProcessLogger

Direct Known Subclasses:

MapReduceFlow, MultiMapReduceFlow

public class BaseMapReduceFlow
extends HadoopFlow

Nested Class Summary

Nested classes/interfaces inherited from class cascading.flow.BaseFlow

cascading.flow.BaseFlow.FlowHolder

Field Summary

Fields

Modifier and Type	Field and Description
protected boolean	deleteSinkOnInit Field deleteSinkOnInit

Fields inherited from class cascading.flow.BaseFlow

completed, flowCanonicalHash, flowDescriptor, flowElementGraph, flowStats, flowStepGraph, platformInfo, sinks, sources, steps, stop, stopJobsOnExit, thread, throwable

Fields inherited from interface cascading.flow.Flow

CASCADING_FLOW_ID

Fields inherited from interface cascading.util.ProcessLogger

NULL

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	BaseMapReduceFlow(cascading.flow.planner.PlatformInfo platformInfo, java.util.Map<java.lang.Object,java.lang.Object> properties, org.apache.hadoop.mapred.JobConf jobConf, java.lang.String name, java.util.Map<java.lang.String,java.lang.String> flowDescriptor, boolean deleteSinkOnInit)
protected	BaseMapReduceFlow(cascading.flow.planner.PlatformInfo platformInfo, java.util.Map<java.lang.Object,java.lang.Object> properties, java.lang.String name, java.util.Map<java.lang.String,java.lang.String> flowDescriptor, boolean deleteSinkOnInit)

Method Summary

Modifier and Type	Method and Description
protected cascading.flow.FlowStep<org.apache.hadoop.mapred.JobConf>	createFlowStep(org.apache.hadoop.mapred.JobConf jobConf, cascading.tap.Tap sink)
protected java.util.Map<java.lang.String,cascading.tap.Tap>	createSinks(org.apache.hadoop.mapred.JobConf jobConf)
protected java.util.Map<java.lang.String,cascading.tap.Tap>	createSources(org.apache.hadoop.mapred.JobConf jobConf)
protected cascading.tap.Tap	createTap(org.apache.hadoop.mapred.JobConf jobConf, org.apache.hadoop.fs.Path path, cascading.tap.SinkMode sinkMode)
protected java.util.Map<java.lang.String,cascading.tap.Tap>	createTraps(org.apache.hadoop.mapred.JobConf jobConf)
protected java.util.Map<java.lang.String,cascading.tap.Tap>	fileInputToTaps(org.apache.hadoop.mapred.JobConf jobConf)
protected java.util.Map<java.lang.String,cascading.tap.Tap>	fileOutputToTaps(org.apache.hadoop.mapred.JobConf jobConf)
protected java.lang.String	makeNameFromPath(java.util.Map<java.lang.String,cascading.tap.Tap> taps, org.apache.hadoop.fs.Path path)
protected cascading.flow.planner.process.FlowStepGraph	makeStepGraph(org.apache.hadoop.mapred.JobConf jobConf)
protected cascading.tap.Tap	toSinkTap(org.apache.hadoop.mapred.JobConf jobConf, java.util.Map<java.lang.String,cascading.tap.Tap> taps, org.apache.hadoop.fs.Path path)
protected cascading.tap.Tap	toSourceTap(org.apache.hadoop.mapred.JobConf jobConf, java.util.Map<java.lang.String,cascading.tap.Tap> taps, org.apache.hadoop.fs.Path path)

Methods inherited from class cascading.flow.hadoop.HadoopFlow

copyToDistributedCache, getConfig, getConfigAsProperties, getConfigCopy, getFlowProcess, getMaxNumParallelSteps, getProperty, getTotalSliceCPUMilliSeconds, initConfig, initFromProperties, internalClean, internalShutdown, internalStart, isPreserveTemporaryFiles, newConfig, registerHadoopShutdownHook, setConfigProperty, stepsAreLocal

Methods inherited from class cascading.flow.BaseFlow

addListener, addPlannerProperties, addSessionProperties, addStepListener, areSinksStale, areSourcesNewer, cleanup, complete, createConfig, createFlowCanonicalHash, createFlowStats, createFlowThread, createPrepareFlowStats, deleteCheckpointsIfNotUpdate, deleteCheckpointsIfReplace, deleteSinks, deleteSinksIfNotUpdate, deleteSinksIfReplace, deleteTrapsIfNotUpdate, deleteTrapsIfReplace, fireOnCompleted, fireOnStarting, fireOnStopping, fireOnThrowable, fireOnThrowable, getCascadeID, getCascadingServices, getCheckpointNames, getCheckpoints, getCheckpointsCollection, getClassPath, getClientState, getEligibleJobsSize, getFieldsFor, getFlowCanonicalHash, getFlowDescriptor, getFlowElementGraph, getFlowSession, getFlowSkipStrategy, getFlowStats, getFlowStepGraph, getFlowSteps, getFlowStepStrategy, getHolder, getID, getJobMapCallables, getName, getPlannerInfo, getPlatformInfo, getRunID, getSink, getSink, getSinkModified, getSinkNames, getSinks, getSinksCollection, getSource, getSourceNames, getSources, getSourcesCollection, getSpawnStrategy, getStats, getSubmitPriority, getTags, getTrapNames, getTraps, getTrapsCollection, handleExecutorShutdown, hasListeners, hasStepListeners, ifStoppingBlockUntilComplete, initialize, initializeChildStats, initializeNewJobsMap, initSteps, internalStopAllJobs, isDebugEnabled, isInfoEnabled, isJobsMapInitialized, isSkipFlow, isStopJobsOnExit, logDebug, logError, logError, logInfo, logWarn, logWarn, logWarn, openSink, openSink, openSource, openSource, openTapForRead, openTapForWrite, openTrap, openTrap, prepare, presentSinkFields, presentSourceFields, registerShutdownHook, removeListener, removeListeners, removeStepListener, resourceExists, retrieveSinkFields, retrieveSourceFields, setCascade, setCheckpoints, setFlowElementGraph, setFlowSkipStrategy, setFlowStepGraph, setFlowStepStrategy, setName, setPlannerInfo, setSinks, setSources, setSpawnStrategy, setSubmitPriority, setTraps, spawnSteps, start, stop, toString, updateJobsMap, updateSchemes, writeDOT, writeStepsDOT

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface cascading.flow.Flow

getSinkEntryStream, getSinkEntryStream, getSinkEntryStream, getSinkEntryStream, getSinkEntryStreamCopy, getSinkEntryStreamCopy, getSinkEntryStreamCopy, getSinkEntryStreamCopy, getSinkTupleStream, getSinkTupleStream, getSinkTupleStream, getSinkTupleStream, getSinkTupleStreamCopy, getSinkTupleStreamCopy, getSinkTupleStreamCopy, getSinkTupleStreamCopy, getSourceEntryStream, getSourceEntryStream, getSourceEntryStreamCopy, getSourceEntryStreamCopy, getSourceTupleStream, getSourceTupleStream, getSourceTupleStreamCopy, getSourceTupleStreamCopy

Field Detail

deleteSinkOnInit

protected boolean deleteSinkOnInit

Field deleteSinkOnInit

Constructor Detail

BaseMapReduceFlow

protected BaseMapReduceFlow(cascading.flow.planner.PlatformInfo platformInfo,
                            java.util.Map<java.lang.Object,java.lang.Object> properties,
                            org.apache.hadoop.mapred.JobConf jobConf,
                            java.lang.String name,
                            java.util.Map<java.lang.String,java.lang.String> flowDescriptor,
                            boolean deleteSinkOnInit)

BaseMapReduceFlow

protected BaseMapReduceFlow(cascading.flow.planner.PlatformInfo platformInfo,
                            java.util.Map<java.lang.Object,java.lang.Object> properties,
                            java.lang.String name,
                            java.util.Map<java.lang.String,java.lang.String> flowDescriptor,
                            boolean deleteSinkOnInit)

Method Detail

makeStepGraph

protected cascading.flow.planner.process.FlowStepGraph makeStepGraph(org.apache.hadoop.mapred.JobConf jobConf)

createFlowStep

protected cascading.flow.FlowStep<org.apache.hadoop.mapred.JobConf> createFlowStep(org.apache.hadoop.mapred.JobConf jobConf,
                                                                                   cascading.tap.Tap sink)

createSources

protected java.util.Map<java.lang.String,cascading.tap.Tap> createSources(org.apache.hadoop.mapred.JobConf jobConf)

fileInputToTaps

protected java.util.Map<java.lang.String,cascading.tap.Tap> fileInputToTaps(org.apache.hadoop.mapred.JobConf jobConf)

toSourceTap

protected cascading.tap.Tap toSourceTap(org.apache.hadoop.mapred.JobConf jobConf,
                                        java.util.Map<java.lang.String,cascading.tap.Tap> taps,
                                        org.apache.hadoop.fs.Path path)

createSinks

protected java.util.Map<java.lang.String,cascading.tap.Tap> createSinks(org.apache.hadoop.mapred.JobConf jobConf)

fileOutputToTaps

protected java.util.Map<java.lang.String,cascading.tap.Tap> fileOutputToTaps(org.apache.hadoop.mapred.JobConf jobConf)

toSinkTap

protected cascading.tap.Tap toSinkTap(org.apache.hadoop.mapred.JobConf jobConf,
                                      java.util.Map<java.lang.String,cascading.tap.Tap> taps,
                                      org.apache.hadoop.fs.Path path)

createTap

protected cascading.tap.Tap createTap(org.apache.hadoop.mapred.JobConf jobConf,
                                      org.apache.hadoop.fs.Path path,
                                      cascading.tap.SinkMode sinkMode)

makeNameFromPath

protected java.lang.String makeNameFromPath(java.util.Map<java.lang.String,cascading.tap.Tap> taps,
                                            org.apache.hadoop.fs.Path path)

createTraps

protected java.util.Map<java.lang.String,cascading.tap.Tap> createTraps(org.apache.hadoop.mapred.JobConf jobConf)