public abstract class ShardedSparkJob extends SparkJob
JavaPairRDD of CountryName to Shardorg.openstreetmap.atlas.utilities.runtime.Command.Flag, org.openstreetmap.atlas.utilities.runtime.Command.Optionality, org.openstreetmap.atlas.utilities.runtime.Command.Switch<T>, org.openstreetmap.atlas.utilities.runtime.Command.SwitchList| Modifier and Type | Field and Description |
|---|---|
static org.openstreetmap.atlas.utilities.runtime.Command.Switch<org.openstreetmap.atlas.utilities.collections.StringList> |
COUNTRIES |
static org.openstreetmap.atlas.utilities.runtime.Command.Switch<java.lang.String> |
COUNTRY_SHAPES |
static org.openstreetmap.atlas.utilities.runtime.Command.Switch<java.lang.String> |
SHARDING_TYPE |
ADDITIONAL_SPARK_OPTIONS, COMPRESS_OUTPUT, EXITED_FILE, FAILED_FILE, INPUT, MASTER, OUTPUT, SAVING_SEPARATOR, SENSITIVE_CONFIGURATION_PATTERN, SPARK_CONTEXT_PROVIDER, SPARK_OPTIONS, STARTED_FILE, STARTED_FOLDER, SUCCESS_FILE| Constructor and Description |
|---|
ShardedSparkJob() |
| Modifier and Type | Method and Description |
|---|---|
void |
start(org.openstreetmap.atlas.utilities.runtime.CommandMap command)
The spark Job
|
abstract void |
start2(org.openstreetmap.atlas.utilities.runtime.CommandMap command,
org.openstreetmap.atlas.geography.sharding.Sharding sharding,
org.openstreetmap.atlas.utilities.maps.MultiMap<java.lang.String,org.openstreetmap.atlas.geography.sharding.Shard> countryToShardMap) |
org.openstreetmap.atlas.utilities.runtime.Command.SwitchList |
switches() |
configuration, configurationMap, getAlternateParallelFolderOutput, getAlternateSubFolderOutput, getContext, getName, input, onRun, output, outputToClean, resource, resource, setContext, splitAndSaveAsHadoopFilepublic static final org.openstreetmap.atlas.utilities.runtime.Command.Switch<org.openstreetmap.atlas.utilities.collections.StringList> COUNTRIES
public static final org.openstreetmap.atlas.utilities.runtime.Command.Switch<java.lang.String> SHARDING_TYPE
public static final org.openstreetmap.atlas.utilities.runtime.Command.Switch<java.lang.String> COUNTRY_SHAPES
public void start(org.openstreetmap.atlas.utilities.runtime.CommandMap command)
SparkJobpublic abstract void start2(org.openstreetmap.atlas.utilities.runtime.CommandMap command,
org.openstreetmap.atlas.geography.sharding.Sharding sharding,
org.openstreetmap.atlas.utilities.maps.MultiMap<java.lang.String,org.openstreetmap.atlas.geography.sharding.Shard> countryToShardMap)