Package cz.o2.proxima.beam.direct.io

Class DirectBatchUnboundedSource

java.lang.Object

org.apache.beam.sdk.io.Source<OutputT>

org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>

cz.o2.proxima.beam.direct.io.DirectBatchUnboundedSource

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

public class DirectBatchUnboundedSource
extends org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>

Source reading from BatchLogReader in unbounded manner.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	DirectBatchUnboundedSource.Checkpoint
static class	DirectBatchUnboundedSource.CheckpointCoder	Use gzip to compress the serialized checkpoint, as it might easily grow in size (batch partitions might contain many files).
private static class	DirectBatchUnboundedSource.StreamElementUnboundedReader

Nested classes/interfaces inherited from class org.apache.beam.sdk.io.UnboundedSource

org.apache.beam.sdk.io.UnboundedSource.CheckpointMark, org.apache.beam.sdk.io.UnboundedSource.UnboundedReader<OutputT extends java.lang.Object>

Nested classes/interfaces inherited from class org.apache.beam.sdk.io.Source

org.apache.beam.sdk.io.Source.Reader<T extends java.lang.Object>

Field Summary

Fields

Modifier and Type	Field	Description
private java.util.List<AttributeDescriptor<?>>	attributes
static java.lang.String	CFG_ENABLE_CHECKPOINT_PARTITION_MERGE
private boolean	enableCheckpointPartitionMerge
private long	endStamp
private java.util.List<Partition>	partitions
private BatchLogReader	reader
private BatchLogReader.Factory<?>	readerFactory
private RepositoryFactory	repositoryFactory
private static long	serialVersionUID
private long	startStamp

Constructor Summary

Constructors

Modifier	Constructor	Description
private	DirectBatchUnboundedSource(DirectBatchUnboundedSource parent, java.util.List<Partition> partitions, long startStamp, long endStamp)
private	DirectBatchUnboundedSource(RepositoryFactory repositoryFactory, BatchLogReader reader, java.util.List<AttributeDescriptor<?>> attributes, long startStamp, long endStamp, java.util.Map<java.lang.String,java.lang.Object> cfg)

Method Summary

Modifier and Type	Method	Description
org.apache.beam.sdk.io.UnboundedSource.UnboundedReader<StreamElement>	createReader(org.apache.beam.sdk.options.PipelineOptions options, DirectBatchUnboundedSource.Checkpoint checkpointMark)
boolean	equals(java.lang.Object o)
private static boolean	getBool(java.lang.String name, java.util.Map<java.lang.String,java.lang.Object> cfg)
org.apache.beam.sdk.coders.Coder<DirectBatchUnboundedSource.Checkpoint>	getCheckpointMarkCoder()
org.apache.beam.sdk.coders.Coder<StreamElement>	getOutputCoder()
int	hashCode()
(package private) static boolean	isEnableCheckpointPartitionMerge(java.util.Map<java.lang.String,java.lang.Object> cfg)
(package private) static java.util.List<Partition>	merge(boolean enabled, java.util.List<Partition> own, java.util.List<Partition> fromCheckpoint)
static DirectBatchUnboundedSource	of(RepositoryFactory factory, BatchLogReader reader, java.util.List<AttributeDescriptor<?>> attrs, long startStamp, long endStamp, java.util.Map<java.lang.String,java.lang.Object> cfg)
private BatchLogReader	reader()
private void	readObject(java.io.ObjectInputStream stream)
java.util.List<? extends org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>>	split(int desiredNumSplits, org.apache.beam.sdk.options.PipelineOptions options)
private void	writeObject(java.io.ObjectOutputStream stream)

Methods inherited from class org.apache.beam.sdk.io.UnboundedSource

requiresDeduping

Methods inherited from class org.apache.beam.sdk.io.Source

getDefaultOutputCoder, populateDisplayData, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, toString, wait, wait, wait

Field Detail

serialVersionUID

private static final long serialVersionUID

See Also:

Constant Field Values

CFG_ENABLE_CHECKPOINT_PARTITION_MERGE

public static final java.lang.String CFG_ENABLE_CHECKPOINT_PARTITION_MERGE

See Also:

Constant Field Values

repositoryFactory

private final RepositoryFactory repositoryFactory

readerFactory

private final BatchLogReader.Factory<?> readerFactory

attributes

private final java.util.List<AttributeDescriptor<?>> attributes

startStamp

private final long startStamp

endStamp

private final long endStamp

enableCheckpointPartitionMerge

private final boolean enableCheckpointPartitionMerge

reader

@Nullable
private transient BatchLogReader reader

partitions

private transient java.util.List<Partition> partitions

Constructor Detail

DirectBatchUnboundedSource

private DirectBatchUnboundedSource(RepositoryFactory repositoryFactory,
                                   BatchLogReader reader,
                                   java.util.List<AttributeDescriptor<?>> attributes,
                                   long startStamp,
                                   long endStamp,
                                   java.util.Map<java.lang.String,java.lang.Object> cfg)

DirectBatchUnboundedSource

private DirectBatchUnboundedSource(DirectBatchUnboundedSource parent,
                                   java.util.List<Partition> partitions,
                                   long startStamp,
                                   long endStamp)

Method Detail

of

public static DirectBatchUnboundedSource of(RepositoryFactory factory,
                                            BatchLogReader reader,
                                            java.util.List<AttributeDescriptor<?>> attrs,
                                            long startStamp,
                                            long endStamp,
                                            java.util.Map<java.lang.String,java.lang.Object> cfg)

isEnableCheckpointPartitionMerge

static boolean isEnableCheckpointPartitionMerge(java.util.Map<java.lang.String,java.lang.Object> cfg)

getBool

private static boolean getBool(java.lang.String name,
                               java.util.Map<java.lang.String,java.lang.Object> cfg)

split

public java.util.List<? extends org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>> split(int desiredNumSplits,
                                                                                                                                         org.apache.beam.sdk.options.PipelineOptions options)

Specified by:

split in class org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>

createReader

public org.apache.beam.sdk.io.UnboundedSource.UnboundedReader<StreamElement> createReader(org.apache.beam.sdk.options.PipelineOptions options,
                                                                                          DirectBatchUnboundedSource.Checkpoint checkpointMark)

Specified by:

createReader in class org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>

merge

static java.util.List<Partition> merge(boolean enabled,
                                       java.util.List<Partition> own,
                                       java.util.List<Partition> fromCheckpoint)

reader

private BatchLogReader reader()

equals

public boolean equals(java.lang.Object o)

Overrides:

equals in class java.lang.Object

hashCode

public int hashCode()

Overrides:

hashCode in class java.lang.Object

getCheckpointMarkCoder

public org.apache.beam.sdk.coders.Coder<DirectBatchUnboundedSource.Checkpoint> getCheckpointMarkCoder()

Specified by:

getCheckpointMarkCoder in class org.apache.beam.sdk.io.UnboundedSource<StreamElement,DirectBatchUnboundedSource.Checkpoint>

getOutputCoder

public org.apache.beam.sdk.coders.Coder<StreamElement> getOutputCoder()

Overrides:

getOutputCoder in class org.apache.beam.sdk.io.Source<StreamElement>

writeObject

private void writeObject(java.io.ObjectOutputStream stream)
                  throws java.io.IOException

Throws:

java.io.IOException

readObject

private void readObject(java.io.ObjectInputStream stream)
                 throws java.io.IOException,
                        java.lang.ClassNotFoundException

Throws:

java.io.IOException

java.lang.ClassNotFoundException