io.warp10.hadoop

Class Warp10InputFormat

java.lang.Object

org.apache.hadoop.mapreduce.InputFormat<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable>

io.warp10.hadoop.Warp10InputFormat

public class Warp10InputFormat
extends org.apache.hadoop.mapreduce.InputFormat<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable>

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	DEFAULT_WARP10_FETCHER_PATH
static java.lang.String	DEFAULT_WARP10_FETCHER_PORT
static java.lang.String	DEFAULT_WARP10_FETCHER_PROTOCOL
static java.lang.String	DEFAULT_WARP10_HTTP_CONNECT_TIMEOUT
static java.lang.String	DEFAULT_WARP10_HTTP_READ_TIMEOUT
static java.lang.String	HTTP_HEADER_COUNT
static java.lang.String	HTTP_HEADER_DEDUP
static java.lang.String	HTTP_HEADER_END
static java.lang.String	HTTP_HEADER_NOW
static java.lang.String	HTTP_HEADER_POSTBOUNDARY
static java.lang.String	HTTP_HEADER_PREBOUNDARY
static java.lang.String	HTTP_HEADER_SAMPLE
static java.lang.String	HTTP_HEADER_SHOW_ERRORS
static java.lang.String	HTTP_HEADER_SKIP
static java.lang.String	HTTP_HEADER_START
static java.lang.String	HTTP_HEADER_STEP
static java.lang.String	HTTP_HEADER_STOP
static java.lang.String	HTTP_HEADER_TIMESPAN
static java.lang.String	HTTP_HEADER_TIMESTEP
static java.lang.String	PROPERTY_WARP10_FETCH_COUNT
static java.lang.String	PROPERTY_WARP10_FETCH_DEDUP
static java.lang.String	PROPERTY_WARP10_FETCH_END
static java.lang.String	PROPERTY_WARP10_FETCH_NOW Now parameter
static java.lang.String	PROPERTY_WARP10_FETCH_POSTBOUNDARY
static java.lang.String	PROPERTY_WARP10_FETCH_PREBOUNDARY
static java.lang.String	PROPERTY_WARP10_FETCH_SAMPLE
static java.lang.String	PROPERTY_WARP10_FETCH_SKIP
static java.lang.String	PROPERTY_WARP10_FETCH_START
static java.lang.String	PROPERTY_WARP10_FETCH_STEP
static java.lang.String	PROPERTY_WARP10_FETCH_STOP
static java.lang.String	PROPERTY_WARP10_FETCH_TIMESPAN Timespan parameter
static java.lang.String	PROPERTY_WARP10_FETCH_TIMESTEP
static java.lang.String	PROPERTY_WARP10_FETCHER_FALLBACKS List of fallback fetchers
static java.lang.String	PROPERTY_WARP10_FETCHER_FALLBACKSONLY Boolean indicating whether to use the fetchers or only the fallbacks
static java.lang.String	PROPERTY_WARP10_FETCHER_PATH URL Path of the fetcher, defaults to "/api/v0/sfetch"
static java.lang.String	PROPERTY_WARP10_FETCHER_PORT Port to use when contacting the fetcher, defaults to 8881
static java.lang.String	PROPERTY_WARP10_FETCHER_PROTOCOL Protocol to use when contacting the fetcher (http or https), defaults to http
static java.lang.String	PROPERTY_WARP10_HTTP_CONNECT_TIMEOUT Connection timeout to the splits and sfetch endpoints, defaults to 10000 ms
static java.lang.String	PROPERTY_WARP10_HTTP_READ_TIMEOUT Read timeout to the splits and sfetch endpoints, defaults to 10000 ms
static java.lang.String	PROPERTY_WARP10_INPUTFORMAT_SUFFIX Suffix as set via the configuration
static java.lang.String	PROPERTY_WARP10_MAX_COMBINED_SPLITS Maximum number of splits to combined into a single split
static java.lang.String	PROPERTY_WARP10_MAX_SPLITS Maximum number of splits we wish to produce
static java.lang.String	PROPERTY_WARP10_SPLITS_ENDPOINT URL of split endpoint
static java.lang.String	PROPERTY_WARP10_SPLITS_SELECTOR GTS Selector
static java.lang.String	PROPERTY_WARP10_SPLITS_TOKEN Token to use for selecting GTS

Constructor Summary

Constructors

Constructor and Description
Warp10InputFormat()
Warp10InputFormat(java.lang.String suffix)

Method Summary

Modifier and Type	Method and Description
org.apache.hadoop.mapreduce.RecordReader<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable>	createRecordReader(org.apache.hadoop.mapreduce.InputSplit split, org.apache.hadoop.mapreduce.TaskAttemptContext context)
static java.lang.String	getProperty(org.apache.hadoop.conf.Configuration conf, java.lang.String suffix, java.lang.String property, java.lang.String defaultValue)
java.util.List<org.apache.hadoop.mapreduce.InputSplit>	getSplits(org.apache.hadoop.mapreduce.JobContext context)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

PROPERTY_WARP10_INPUTFORMAT_SUFFIX

public static final java.lang.String PROPERTY_WARP10_INPUTFORMAT_SUFFIX

Suffix as set via the configuration

See Also:

Constant Field Values

PROPERTY_WARP10_SPLITS_ENDPOINT

public static final java.lang.String PROPERTY_WARP10_SPLITS_ENDPOINT

URL of split endpoint

See Also:

Constant Field Values

PROPERTY_WARP10_FETCHER_FALLBACKS

public static final java.lang.String PROPERTY_WARP10_FETCHER_FALLBACKS

List of fallback fetchers

See Also:

Constant Field Values

PROPERTY_WARP10_FETCHER_FALLBACKSONLY

public static final java.lang.String PROPERTY_WARP10_FETCHER_FALLBACKSONLY

Boolean indicating whether to use the fetchers or only the fallbacks

See Also:

Constant Field Values

PROPERTY_WARP10_FETCHER_PROTOCOL

public static final java.lang.String PROPERTY_WARP10_FETCHER_PROTOCOL

Protocol to use when contacting the fetcher (http or https), defaults to http

See Also:

Constant Field Values

DEFAULT_WARP10_FETCHER_PROTOCOL

public static final java.lang.String DEFAULT_WARP10_FETCHER_PROTOCOL

See Also:

Constant Field Values

PROPERTY_WARP10_FETCHER_PORT

public static final java.lang.String PROPERTY_WARP10_FETCHER_PORT

Port to use when contacting the fetcher, defaults to 8881

See Also:

Constant Field Values

DEFAULT_WARP10_FETCHER_PORT

public static final java.lang.String DEFAULT_WARP10_FETCHER_PORT

See Also:

Constant Field Values

PROPERTY_WARP10_FETCHER_PATH

public static final java.lang.String PROPERTY_WARP10_FETCHER_PATH

URL Path of the fetcher, defaults to "/api/v0/sfetch"

See Also:

Constant Field Values

DEFAULT_WARP10_FETCHER_PATH

public static final java.lang.String DEFAULT_WARP10_FETCHER_PATH

See Also:

Constant Field Values

PROPERTY_WARP10_SPLITS_SELECTOR

public static final java.lang.String PROPERTY_WARP10_SPLITS_SELECTOR

GTS Selector

See Also:

Constant Field Values

PROPERTY_WARP10_SPLITS_TOKEN

public static final java.lang.String PROPERTY_WARP10_SPLITS_TOKEN

Token to use for selecting GTS

See Also:

Constant Field Values

PROPERTY_WARP10_HTTP_CONNECT_TIMEOUT

public static final java.lang.String PROPERTY_WARP10_HTTP_CONNECT_TIMEOUT

Connection timeout to the splits and sfetch endpoints, defaults to 10000 ms

See Also:

Constant Field Values

DEFAULT_WARP10_HTTP_CONNECT_TIMEOUT

public static final java.lang.String DEFAULT_WARP10_HTTP_CONNECT_TIMEOUT

See Also:

Constant Field Values

PROPERTY_WARP10_HTTP_READ_TIMEOUT

public static final java.lang.String PROPERTY_WARP10_HTTP_READ_TIMEOUT

Read timeout to the splits and sfetch endpoints, defaults to 10000 ms

See Also:

Constant Field Values

DEFAULT_WARP10_HTTP_READ_TIMEOUT

public static final java.lang.String DEFAULT_WARP10_HTTP_READ_TIMEOUT

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_NOW

public static final java.lang.String PROPERTY_WARP10_FETCH_NOW

Now parameter

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_TIMESPAN

public static final java.lang.String PROPERTY_WARP10_FETCH_TIMESPAN

Timespan parameter

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_START

public static final java.lang.String PROPERTY_WARP10_FETCH_START

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_STOP

public static final java.lang.String PROPERTY_WARP10_FETCH_STOP

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_END

public static final java.lang.String PROPERTY_WARP10_FETCH_END

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_COUNT

public static final java.lang.String PROPERTY_WARP10_FETCH_COUNT

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_DEDUP

public static final java.lang.String PROPERTY_WARP10_FETCH_DEDUP

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_SKIP

public static final java.lang.String PROPERTY_WARP10_FETCH_SKIP

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_SAMPLE

public static final java.lang.String PROPERTY_WARP10_FETCH_SAMPLE

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_STEP

public static final java.lang.String PROPERTY_WARP10_FETCH_STEP

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_TIMESTEP

public static final java.lang.String PROPERTY_WARP10_FETCH_TIMESTEP

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_PREBOUNDARY

public static final java.lang.String PROPERTY_WARP10_FETCH_PREBOUNDARY

See Also:

Constant Field Values

PROPERTY_WARP10_FETCH_POSTBOUNDARY

public static final java.lang.String PROPERTY_WARP10_FETCH_POSTBOUNDARY

See Also:

Constant Field Values

PROPERTY_WARP10_MAX_COMBINED_SPLITS

public static final java.lang.String PROPERTY_WARP10_MAX_COMBINED_SPLITS

Maximum number of splits to combined into a single split

See Also:

Constant Field Values

PROPERTY_WARP10_MAX_SPLITS

public static final java.lang.String PROPERTY_WARP10_MAX_SPLITS

Maximum number of splits we wish to produce

See Also:

Constant Field Values

HTTP_HEADER_NOW

public static final java.lang.String HTTP_HEADER_NOW

See Also:

Constant Field Values

HTTP_HEADER_TIMESPAN

public static final java.lang.String HTTP_HEADER_TIMESPAN

See Also:

Constant Field Values

HTTP_HEADER_START

public static final java.lang.String HTTP_HEADER_START

See Also:

Constant Field Values

HTTP_HEADER_STOP

public static final java.lang.String HTTP_HEADER_STOP

See Also:

Constant Field Values

HTTP_HEADER_END

public static final java.lang.String HTTP_HEADER_END

See Also:

Constant Field Values

HTTP_HEADER_DEDUP

public static final java.lang.String HTTP_HEADER_DEDUP

See Also:

Constant Field Values

HTTP_HEADER_COUNT

public static final java.lang.String HTTP_HEADER_COUNT

See Also:

Constant Field Values

HTTP_HEADER_SKIP

public static final java.lang.String HTTP_HEADER_SKIP

See Also:

Constant Field Values

HTTP_HEADER_SAMPLE

public static final java.lang.String HTTP_HEADER_SAMPLE

See Also:

Constant Field Values

HTTP_HEADER_STEP

public static final java.lang.String HTTP_HEADER_STEP

See Also:

Constant Field Values

HTTP_HEADER_PREBOUNDARY

public static final java.lang.String HTTP_HEADER_PREBOUNDARY

See Also:

Constant Field Values

HTTP_HEADER_POSTBOUNDARY

public static final java.lang.String HTTP_HEADER_POSTBOUNDARY

See Also:

Constant Field Values

HTTP_HEADER_TIMESTEP

public static final java.lang.String HTTP_HEADER_TIMESTEP

See Also:

Constant Field Values

HTTP_HEADER_SHOW_ERRORS

public static final java.lang.String HTTP_HEADER_SHOW_ERRORS

See Also:

Constant Field Values

Constructor Detail

Warp10InputFormat

public Warp10InputFormat(java.lang.String suffix)

Warp10InputFormat

public Warp10InputFormat()

Method Detail

getSplits

public java.util.List<org.apache.hadoop.mapreduce.InputSplit> getSplits(org.apache.hadoop.mapreduce.JobContext context)
                                                                 throws java.io.IOException

Specified by:

getSplits in class org.apache.hadoop.mapreduce.InputFormat<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable>

Throws:

java.io.IOException

createRecordReader

public org.apache.hadoop.mapreduce.RecordReader<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable> createRecordReader(org.apache.hadoop.mapreduce.InputSplit split,
                                                                                                                                 org.apache.hadoop.mapreduce.TaskAttemptContext context)
                                                                                                                          throws java.io.IOException

Specified by:

createRecordReader in class org.apache.hadoop.mapreduce.InputFormat<org.apache.hadoop.io.Text,org.apache.hadoop.io.BytesWritable>

Throws:

java.io.IOException

getProperty

public static java.lang.String getProperty(org.apache.hadoop.conf.Configuration conf,
                                           java.lang.String suffix,
                                           java.lang.String property,
                                           java.lang.String defaultValue)