Package cz.o2.proxima.beam.core

Class BeamDataOperator

java.lang.Object

cz.o2.proxima.beam.core.BeamDataOperator

All Implemented Interfaces:

DataOperator, java.lang.AutoCloseable

public class BeamDataOperator
extends java.lang.Object
implements DataOperator

A DataOperator for Apache Beam transformations.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	BeamDataOperator.BatchSnapshotDescriptor
private static class	BeamDataOperator.BatchUpdatesDescriptor
private static interface	BeamDataOperator.PCollectionDescriptor
private static interface	BeamDataOperator.PCollectionFactoryFromDescriptor<T extends BeamDataOperator.PCollectionDescriptor>
private static class	BeamDataOperator.StreamDescriptor

Field Summary

Fields

Modifier and Type	Field	Description
private java.util.Map<AttributeFamilyDescriptor,DataAccessor>	accessorMap
private java.util.Map<BeamDataOperator.PCollectionDescriptor,org.apache.beam.sdk.values.PCollection<StreamElement>>	createdStreamsMap
private DirectDataOperator	direct
private DataAccessorLoader<BeamDataOperator,DataAccessor,DataAccessorFactory>	loader
private Repository	repo
private java.util.Set<org.apache.beam.sdk.Pipeline>	typesRegistered

Constructor Summary

Constructors

Constructor	Description
BeamDataOperator(Repository repo)

Method Summary

Modifier and Type	Method	Description
private DataAccessor	accessorFor(AttributeFamilyDescriptor family)
void	close()
private DataAccessor	createAccessorFor(AttributeFamilyDescriptor family)
private java.util.function.Supplier<java.lang.IllegalArgumentException>	failNotFound(AttributeDescriptor<?>[] attrs, java.lang.String accessorType)
private org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<StreamElement>,org.apache.beam.sdk.values.PCollection<StreamElement>>	filterAttrs(AttributeDescriptor<?>[] attrs)
private java.util.stream.Stream<DataAccessor>	findSuitableAccessors(java.util.function.Predicate<AttributeFamilyDescriptor> predicate, java.lang.String accessorType, AttributeDescriptor<?>[] attrs)
private java.util.stream.Stream<Pair<AttributeDescriptor<?>,java.util.Optional<AttributeFamilyDescriptor>>>	findSuitableFamilies(java.util.function.Predicate<AttributeFamilyDescriptor> predicate, AttributeDescriptor<?>[] attrs)
DataAccessor	getAccessorFor(AttributeFamilyDescriptor family)	Get DataAccessor for given AttributeFamilyDescriptor.
org.apache.beam.sdk.values.PCollection<StreamElement>	getBatchSnapshot(org.apache.beam.sdk.Pipeline pipeline, long fromStamp, long untilStamp, AttributeDescriptor<?>... attrs)	Create PCollection from snapshot of given attributes.
org.apache.beam.sdk.values.PCollection<StreamElement>	getBatchSnapshot(org.apache.beam.sdk.Pipeline pipeline, AttributeDescriptor<?>... attrs)	Create PCollection from snapshot of given attributes.
org.apache.beam.sdk.values.PCollection<StreamElement>	getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline, long startStamp, long endStamp, boolean asStream, AttributeDescriptor<?>... attrs)	Create PCollection from updates to given attributes with given time range.
org.apache.beam.sdk.values.PCollection<StreamElement>	getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline, long startStamp, long endStamp, AttributeDescriptor<?>... attrs)	Create PCollection from updates to given attributes with given time range.
org.apache.beam.sdk.values.PCollection<StreamElement>	getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline, AttributeDescriptor<?>... attrs)	Create PCollection from updates to given attributes.
DirectDataOperator	getDirect()
private <T extends BeamDataOperator.PCollectionDescriptor> org.apache.beam.sdk.values.PCollection<StreamElement>	getOrCreatePCollection(T desc, boolean cacheable, BeamDataOperator.PCollectionFactoryFromDescriptor<T> factory)
Repository	getRepository()
(package private) org.apache.beam.sdk.values.PCollection<StreamElement>	getStream(java.lang.String name, org.apache.beam.sdk.Pipeline pipeline, Position position, boolean stopAtCurrent, boolean useEventTime, long limit, AttributeDescriptor<?>... attrs)	Create PCollection in given Pipeline from commit log for given attributes limiting number of elements read.
org.apache.beam.sdk.values.PCollection<StreamElement>	getStream(java.lang.String name, org.apache.beam.sdk.Pipeline pipeline, Position position, boolean stopAtCurrent, boolean useEventTime, AttributeDescriptor<?>... attrs)	Create PCollection in given Pipeline from commit log for given attributes.
org.apache.beam.sdk.values.PCollection<StreamElement>	getStream(org.apache.beam.sdk.Pipeline pipeline, Position position, boolean stopAtCurrent, boolean useEventTime, AttributeDescriptor<?>... attrs)	Create PCollection in given Pipeline from commit log for given attributes.
boolean	hasDirect()
private void	registerTypesFor(org.apache.beam.sdk.Pipeline pipeline)
void	reload()
private boolean	typesRegisteredFor(org.apache.beam.sdk.Pipeline pipeline)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

repo

private final Repository repo

direct

@Nullable
private final DirectDataOperator direct

loader

private final DataAccessorLoader<BeamDataOperator,DataAccessor,DataAccessorFactory> loader

accessorMap

private final java.util.Map<AttributeFamilyDescriptor,DataAccessor> accessorMap

createdStreamsMap

private final java.util.Map<BeamDataOperator.PCollectionDescriptor,org.apache.beam.sdk.values.PCollection<StreamElement>> createdStreamsMap

typesRegistered

private final java.util.Set<org.apache.beam.sdk.Pipeline> typesRegistered

Constructor Detail

BeamDataOperator

BeamDataOperator(Repository repo)

Method Detail

close

public void close()

Specified by:

close in interface java.lang.AutoCloseable

Specified by:

close in interface DataOperator

reload

public void reload()

Specified by:

reload in interface DataOperator

getStream

@SafeVarargs
public final org.apache.beam.sdk.values.PCollection<StreamElement> getStream(org.apache.beam.sdk.Pipeline pipeline,
                                                                             Position position,
                                                                             boolean stopAtCurrent,
                                                                             boolean useEventTime,
                                                                             AttributeDescriptor<?>... attrs)

Create PCollection in given Pipeline from commit log for given attributes.

Parameters:

pipeline - the Pipeline to create PCollection in.

position - position in commit log to read from

stopAtCurrent - true to stop at recent data

useEventTime - true to use event time

attrs - the attributes to create PCollection for

Returns:

the PCollection

getStream

@SafeVarargs
public final org.apache.beam.sdk.values.PCollection<StreamElement> getStream(@Nullable
                                                                             java.lang.String name,
                                                                             org.apache.beam.sdk.Pipeline pipeline,
                                                                             Position position,
                                                                             boolean stopAtCurrent,
                                                                             boolean useEventTime,
                                                                             AttributeDescriptor<?>... attrs)

Create PCollection in given Pipeline from commit log for given attributes.

Parameters:

name - name of the consumer

pipeline - the Pipeline to create PCollection in.

position - position in commit log to read from

stopAtCurrent - true to stop at recent data

useEventTime - true to use event time

attrs - the attributes to create PCollection for

Returns:

the PCollection

getStream

@SafeVarargs
final org.apache.beam.sdk.values.PCollection<StreamElement> getStream(@Nullable
                                                                      java.lang.String name,
                                                                      org.apache.beam.sdk.Pipeline pipeline,
                                                                      Position position,
                                                                      boolean stopAtCurrent,
                                                                      boolean useEventTime,
                                                                      long limit,
                                                                      AttributeDescriptor<?>... attrs)

Create PCollection in given Pipeline from commit log for given attributes limiting number of elements read.

Parameters:

name - name of the consumer

pipeline - the Pipeline to create PCollection in.

position - position in commit log to read from

stopAtCurrent - true to stop at recent data

useEventTime - true to use event time

limit - number of elements to read from the source

attrs - the attributes to create PCollection for

Returns:

the PCollection

getBatchUpdates

@SafeVarargs
public final org.apache.beam.sdk.values.PCollection<StreamElement> getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline,
                                                                                   AttributeDescriptor<?>... attrs)

Create PCollection from updates to given attributes.

Parameters:

pipeline - Pipeline to create the PCollection in

attrs - attributes to read updates for

Returns:

the PCollection

getBatchUpdates

@SafeVarargs
public final org.apache.beam.sdk.values.PCollection<StreamElement> getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline,
                                                                                   long startStamp,
                                                                                   long endStamp,
                                                                                   AttributeDescriptor<?>... attrs)

Create PCollection from updates to given attributes with given time range.

Parameters:

pipeline - Pipeline to create the PCollection in

startStamp - timestamp (inclusive) of first update taken into account

endStamp - timestamp (exclusive) of last update taken into account

attrs - attributes to read updates for

Returns:

the PCollection

getBatchUpdates

@SafeVarargs
public final org.apache.beam.sdk.values.PCollection<StreamElement> getBatchUpdates(org.apache.beam.sdk.Pipeline pipeline,
                                                                                   long startStamp,
                                                                                   long endStamp,
                                                                                   boolean asStream,
                                                                                   AttributeDescriptor<?>... attrs)

Create PCollection from updates to given attributes with given time range.

Parameters:

pipeline - Pipeline to create the PCollection in

startStamp - timestamp (inclusive) of first update taken into account

endStamp - timestamp (exclusive) of last update taken into account

asStream - create PCollection that is suitable for streaming processing (i.e. can update watermarks before end of input)

attrs - attributes to read updates for

Returns:

the PCollection

getBatchSnapshot

public final org.apache.beam.sdk.values.PCollection<StreamElement> getBatchSnapshot(org.apache.beam.sdk.Pipeline pipeline,
                                                                                    AttributeDescriptor<?>... attrs)

Create PCollection from snapshot of given attributes. The snapshot is either read from available storage or created by reduction of updates.

Parameters:

pipeline - Pipeline to create the PCollection in

attrs - attributes to read snapshot for

Returns:

the PCollection

getBatchSnapshot

public final org.apache.beam.sdk.values.PCollection<StreamElement> getBatchSnapshot(org.apache.beam.sdk.Pipeline pipeline,
                                                                                    long fromStamp,
                                                                                    long untilStamp,
                                                                                    AttributeDescriptor<?>... attrs)

Create PCollection from snapshot of given attributes. The snapshot is either read from available storage or created by reduction of updates.

Parameters:

pipeline - Pipeline to create the PCollection in

fromStamp - ignore updates older than this stamp

untilStamp - read only updates older than this timestamp (i.e. if this method was called at the given timestamp)

attrs - attributes to read snapshot for

Returns:

the PCollection

getAccessorFor

public DataAccessor getAccessorFor(AttributeFamilyDescriptor family)

Get DataAccessor for given AttributeFamilyDescriptor.

Needed for low-level access handling.

Parameters:

family - descriptor of family to retrieve accessor for

Returns:

DataAccessor for given family

findSuitableAccessors

private java.util.stream.Stream<DataAccessor> findSuitableAccessors(java.util.function.Predicate<AttributeFamilyDescriptor> predicate,
                                                                    java.lang.String accessorType,
                                                                    AttributeDescriptor<?>[] attrs)

findSuitableFamilies

private java.util.stream.Stream<Pair<AttributeDescriptor<?>,java.util.Optional<AttributeFamilyDescriptor>>> findSuitableFamilies(java.util.function.Predicate<AttributeFamilyDescriptor> predicate,
                                                                                                                                       AttributeDescriptor<?>[] attrs)

accessorFor

private DataAccessor accessorFor(AttributeFamilyDescriptor family)

createAccessorFor

private DataAccessor createAccessorFor(AttributeFamilyDescriptor family)

getRepository

public Repository getRepository()

Specified by:

getRepository in interface DataOperator

getDirect

public DirectDataOperator getDirect()

hasDirect

public boolean hasDirect()

filterAttrs

private org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<StreamElement>,org.apache.beam.sdk.values.PCollection<StreamElement>> filterAttrs(AttributeDescriptor<?>[] attrs)

failNotFound

private java.util.function.Supplier<java.lang.IllegalArgumentException> failNotFound(AttributeDescriptor<?>[] attrs,
                                                                                     java.lang.String accessorType)

getOrCreatePCollection

private <T extends BeamDataOperator.PCollectionDescriptor> org.apache.beam.sdk.values.PCollection<StreamElement> getOrCreatePCollection(T desc,
                                                                                                                                        boolean cacheable,
                                                                                                                                        BeamDataOperator.PCollectionFactoryFromDescriptor<T> factory)

registerTypesFor

private void registerTypesFor(org.apache.beam.sdk.Pipeline pipeline)

typesRegisteredFor

private boolean typesRegisteredFor(org.apache.beam.sdk.Pipeline pipeline)