Package org.openstreetmap.atlas.generator

Class AtlasGeneratorHelper

java.lang.Object

org.openstreetmap.atlas.generator.AtlasGeneratorHelper

All Implemented Interfaces:

java.io.Serializable

public final class AtlasGeneratorHelper
extends java.lang.Object
implements java.io.Serializable

Utility class for AtlasGenerator.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
protected static class	AtlasGeneratorHelper.NamedAtlasStatistics

Field Summary

Fields

Modifier and Type	Field	Description
static java.lang.String	ERROR_MESSAGE
static java.lang.String	FINISHED_MESSAGE
static java.lang.String	MEMORY_MESSAGE
static java.lang.String	STARTED_MESSAGE

Method Summary

Modifier and Type	Method	Description
static java.util.function.Function<org.openstreetmap.atlas.geography.sharding.Shard,java.util.Optional<org.openstreetmap.atlas.geography.atlas.Atlas>>	atlasFetcher(HadoopAtlasFileCache subAtlasCache, HadoopAtlasFileCache atlasCache, java.lang.String countryBeingSliced, org.openstreetmap.atlas.geography.sharding.Shard initialShard)
static java.util.function.Function<org.openstreetmap.atlas.geography.sharding.Shard,java.util.Optional<org.openstreetmap.atlas.geography.atlas.Atlas>>	atlasFetcher(HadoopAtlasFileCache lineSlicedSubAtlasCache, org.openstreetmap.atlas.geography.atlas.Atlas initialShardAtlas, org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap boundaries, java.lang.String countryBeingSliced, org.openstreetmap.atlas.geography.sharding.Shard initialShard)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,java.util.List<org.openstreetmap.atlas.geography.atlas.change.FeatureChange>>	computeAtlasDiff(java.util.Map<java.lang.String,java.lang.String> sparkContext, java.lang.String previousOutputForDelta)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics>	generateAtlasStatistics(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding)
protected static org.apache.spark.api.java.function.PairFunction<org.openstreetmap.atlas.generator.AtlasGenerationTask,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>	generateRawAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries, java.util.Map<java.lang.String,java.lang.String> sparkContext, org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions, PbfContext pbfContext, SlippyTilePersistenceScheme atlasScheme)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics>,java.lang.String,AtlasGeneratorHelper.NamedAtlasStatistics>	groupAtlasStatisticsByCountry()
protected static org.apache.spark.api.java.function.Function2<AtlasGeneratorHelper.NamedAtlasStatistics,AtlasGeneratorHelper.NamedAtlasStatistics,AtlasGeneratorHelper.NamedAtlasStatistics>	reduceAtlasStatistics()
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>	sectionAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries, org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding, java.util.Map<java.lang.String,java.lang.String> sparkContext, org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions, java.lang.String edgeSubAtlasPath, java.lang.String slicedAtlasPath, SlippyTilePersistenceScheme atlasScheme)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>	sliceAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries, org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions, org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding, java.lang.String lineSlicedSubAtlasPath, SlippyTilePersistenceScheme atlasScheme, java.util.Map<java.lang.String,java.lang.String> sparkContext)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>	subatlas(java.util.function.Predicate<org.openstreetmap.atlas.geography.atlas.items.AtlasEntity> filter, org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType cutType)
protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>	subatlas(org.openstreetmap.atlas.utilities.configuration.ConfiguredFilter filter, org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType cutType)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

STARTED_MESSAGE

public static final java.lang.String STARTED_MESSAGE

See Also:

Constant Field Values

FINISHED_MESSAGE

public static final java.lang.String FINISHED_MESSAGE

See Also:

Constant Field Values

ERROR_MESSAGE

public static final java.lang.String ERROR_MESSAGE

See Also:

Constant Field Values

MEMORY_MESSAGE

public static final java.lang.String MEMORY_MESSAGE

See Also:

Constant Field Values

Method Detail

atlasFetcher

public static java.util.function.Function<org.openstreetmap.atlas.geography.sharding.Shard,java.util.Optional<org.openstreetmap.atlas.geography.atlas.Atlas>> atlasFetcher(HadoopAtlasFileCache lineSlicedSubAtlasCache,
                                                                                                                                                                                 org.openstreetmap.atlas.geography.atlas.Atlas initialShardAtlas,
                                                                                                                                                                                 org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap boundaries,
                                                                                                                                                                                 java.lang.String countryBeingSliced,
                                                                                                                                                                                 org.openstreetmap.atlas.geography.sharding.Shard initialShard)

atlasFetcher

public static java.util.function.Function<org.openstreetmap.atlas.geography.sharding.Shard,java.util.Optional<org.openstreetmap.atlas.geography.atlas.Atlas>> atlasFetcher(HadoopAtlasFileCache subAtlasCache,
                                                                                                                                                                                 HadoopAtlasFileCache atlasCache,
                                                                                                                                                                                 java.lang.String countryBeingSliced,
                                                                                                                                                                                 org.openstreetmap.atlas.geography.sharding.Shard initialShard)

computeAtlasDiff

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,java.util.List<org.openstreetmap.atlas.geography.atlas.change.FeatureChange>> computeAtlasDiff(java.util.Map<java.lang.String,java.lang.String> sparkContext,
                                                                                                                                                                                                                                                                                java.lang.String previousOutputForDelta)

Parameters:

sparkContext - Spark context (or configuration) as a key-value map

previousOutputForDelta - Previous Atlas generation delta output location

Returns:

A Spark PairFlatMapFunction that takes a tuple of a country shard name and atlas file and returns all the AtlasDiff for the country

generateAtlasStatistics

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics> generateAtlasStatistics(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding)

Parameters:

sharding - The sharding tree

Returns:

a Spark PairFunction that processes a shard to Atlas tuple, and constructs a AtlasStatistics for each shard.

generateRawAtlas

protected static org.apache.spark.api.java.function.PairFunction<org.openstreetmap.atlas.generator.AtlasGenerationTask,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas> generateRawAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries,
                                                                                                                                                                                                                    java.util.Map<java.lang.String,java.lang.String> sparkContext,
                                                                                                                                                                                                                    org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions,
                                                                                                                                                                                                                    PbfContext pbfContext,
                                                                                                                                                                                                                    SlippyTilePersistenceScheme atlasScheme)

Parameters:

boundaries - The CountryBoundaryMap to use for pbf to atlas generation

sparkContext - Spark context (or configuration) as a key-value map

loadingOptions - The basic required properties to create an AtlasLoadingOption

pbfContext - The context explaining where to find the PBFs

atlasScheme - The folder structure of the output atlas

Returns:

a Spark PairFunction that processes an AtlasGenerationTask, loads the PBF for the task's shard, generates the raw atlas for the shard and outputs a shard name to raw atlas tuple.

groupAtlasStatisticsByCountry

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.statistics.AtlasStatistics>,java.lang.String,AtlasGeneratorHelper.NamedAtlasStatistics> groupAtlasStatisticsByCountry()

reduceAtlasStatistics

protected static org.apache.spark.api.java.function.Function2<AtlasGeneratorHelper.NamedAtlasStatistics,AtlasGeneratorHelper.NamedAtlasStatistics,AtlasGeneratorHelper.NamedAtlasStatistics> reduceAtlasStatistics()

sectionAtlas

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas> sectionAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries,
                                                                                                                                                                                                                                             org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding,
                                                                                                                                                                                                                                             java.util.Map<java.lang.String,java.lang.String> sparkContext,
                                                                                                                                                                                                                                             org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions,
                                                                                                                                                                                                                                             java.lang.String edgeSubAtlasPath,
                                                                                                                                                                                                                                             java.lang.String slicedAtlasPath,
                                                                                                                                                                                                                                             SlippyTilePersistenceScheme atlasScheme)

Parameters:

boundaries - The CountryBoundaryMap required to create an AtlasLoadingOption

sharding - The Sharding strategy

sparkContext - Spark context (or configuration) as a key-value map

loadingOptions - The basic required properties to create an AtlasLoadingOption

edgeSubAtlasPath - The path where the edge-only sub atlas files were saved

slicedAtlasPath - The path where the sliced raw atlas files were saved

atlasScheme - The folder structure of the output atlas

Returns:

a Spark PairFunction that processes a tuple of shard-name and sliced raw atlas, sections the sliced raw atlas and returns the final sectioned (and sliced) raw atlas for that shard name.

sliceAtlas

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas> sliceAtlas(org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap> boundaries,
                                                                                                                                                                                                                                           org.apache.spark.broadcast.Broadcast<java.util.Map<java.lang.String,java.lang.String>> loadingOptions,
                                                                                                                                                                                                                                           org.apache.spark.broadcast.Broadcast<org.openstreetmap.atlas.geography.sharding.Sharding> sharding,
                                                                                                                                                                                                                                           java.lang.String lineSlicedSubAtlasPath,
                                                                                                                                                                                                                                           SlippyTilePersistenceScheme atlasScheme,
                                                                                                                                                                                                                                           java.util.Map<java.lang.String,java.lang.String> sparkContext)

subatlas

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas> subatlas(org.openstreetmap.atlas.utilities.configuration.ConfiguredFilter filter,
                                                                                                                                                                                                                                         org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType cutType)

subatlas

protected static org.apache.spark.api.java.function.PairFunction<scala.Tuple2<java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas>,java.lang.String,org.openstreetmap.atlas.geography.atlas.Atlas> subatlas(java.util.function.Predicate<org.openstreetmap.atlas.geography.atlas.items.AtlasEntity> filter,
                                                                                                                                                                                                                                         org.openstreetmap.atlas.geography.atlas.sub.AtlasCutType cutType)