Package cz.o2.proxima.beam.direct.io

Class CommitLogRead.AbstractCommitLogReadFn

java.lang.Object

org.apache.beam.sdk.transforms.DoFn<byte[],StreamElement>

cz.o2.proxima.beam.direct.io.CommitLogRead.AbstractCommitLogReadFn

All Implemented Interfaces:

java.io.Serializable, org.apache.beam.sdk.transforms.display.HasDisplayData

Direct Known Subclasses:

CommitLogRead.BoundedCommitLogReadFn, CommitLogRead.UnboundedCommitLogReadFn

Enclosing class:

CommitLogRead

private abstract class CommitLogRead.AbstractCommitLogReadFn
extends org.apache.beam.sdk.transforms.DoFn<byte[],StreamElement>

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.beam.sdk.transforms.DoFn

org.apache.beam.sdk.transforms.DoFn.AlwaysFetched, org.apache.beam.sdk.transforms.DoFn.BoundedPerElement, org.apache.beam.sdk.transforms.DoFn.BundleFinalizer, org.apache.beam.sdk.transforms.DoFn.Element, org.apache.beam.sdk.transforms.DoFn.FieldAccess, org.apache.beam.sdk.transforms.DoFn.FinishBundle, org.apache.beam.sdk.transforms.DoFn.FinishBundleContext, org.apache.beam.sdk.transforms.DoFn.GetInitialRestriction, org.apache.beam.sdk.transforms.DoFn.GetInitialWatermarkEstimatorState, org.apache.beam.sdk.transforms.DoFn.GetRestrictionCoder, org.apache.beam.sdk.transforms.DoFn.GetSize, org.apache.beam.sdk.transforms.DoFn.GetWatermarkEstimatorStateCoder, org.apache.beam.sdk.transforms.DoFn.Key, org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver, org.apache.beam.sdk.transforms.DoFn.NewTracker, org.apache.beam.sdk.transforms.DoFn.NewWatermarkEstimator, org.apache.beam.sdk.transforms.DoFn.OnTimer, org.apache.beam.sdk.transforms.DoFn.OnTimerContext, org.apache.beam.sdk.transforms.DoFn.OnTimerFamily, org.apache.beam.sdk.transforms.DoFn.OnWindowExpiration, org.apache.beam.sdk.transforms.DoFn.OnWindowExpirationContext, org.apache.beam.sdk.transforms.DoFn.OutputReceiver<T extends java.lang.Object>, org.apache.beam.sdk.transforms.DoFn.ProcessContext, org.apache.beam.sdk.transforms.DoFn.ProcessContinuation, org.apache.beam.sdk.transforms.DoFn.ProcessElement, org.apache.beam.sdk.transforms.DoFn.RequiresStableInput, org.apache.beam.sdk.transforms.DoFn.RequiresTimeSortedInput, org.apache.beam.sdk.transforms.DoFn.Restriction, org.apache.beam.sdk.transforms.DoFn.Setup, org.apache.beam.sdk.transforms.DoFn.SideInput, org.apache.beam.sdk.transforms.DoFn.SplitRestriction, org.apache.beam.sdk.transforms.DoFn.StartBundle, org.apache.beam.sdk.transforms.DoFn.StartBundleContext, org.apache.beam.sdk.transforms.DoFn.StateId, org.apache.beam.sdk.transforms.DoFn.Teardown, org.apache.beam.sdk.transforms.DoFn.TimerFamily, org.apache.beam.sdk.transforms.DoFn.TimerId, org.apache.beam.sdk.transforms.DoFn.Timestamp, org.apache.beam.sdk.transforms.DoFn.TruncateRestriction, org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement, org.apache.beam.sdk.transforms.DoFn.WatermarkEstimatorState, org.apache.beam.sdk.transforms.DoFn.WindowedContext

Field Summary

Fields

Modifier and Type	Field	Description
private boolean	externalizableOffsets
protected long	limit
protected java.lang.String	name
protected java.util.Map<java.lang.Integer,BlockingQueueLogObserver.CommitLogObserver>	observers
protected java.util.Map<java.lang.Integer,Offset>	partitionToSeekedOffset
protected Position	position
protected CommitLogReader.Factory<?>	readerFactory
protected RepositoryFactory	repositoryFactory
protected java.util.Map<java.lang.Integer,ObserveHandle>	runningObserves

Constructor Summary

Constructors

Constructor	Description
AbstractCommitLogReadFn(java.lang.String name, Position position, long limit, RepositoryFactory repositoryFactory, CommitLogReader.Factory<?> readerFactory)

Method Summary

Modifier and Type	Method	Description
protected void	closeHandle(int part, boolean nack)
private void	closeHandleIfUnmatchingOffsets(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<OffsetRestrictionTracker.OffsetRange,Offset> tracker, Partition part, BlockingQueueLogObserver.CommitLogObserver observer)
org.joda.time.Instant	getInitialWatermarkEstimatorState()
org.apache.beam.sdk.coders.Coder<OffsetRestrictionTracker.OffsetRange>	getRestrictionCoder()
org.apache.beam.sdk.coders.Coder<org.joda.time.Instant>	getWatermarkEstimatorStateCoder()
org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.Manual	newWatermarkEstimator(org.joda.time.Instant initialWatemark)
(package private) abstract ObserveHandle	observeBulkOffsets(OffsetRestrictionTracker.OffsetRange restriction, CommitLogReader reader, BlockingQueueLogObserver.CommitLogObserver observer)
(package private) abstract ObserveHandle	observeBulkPartitions(java.lang.String name, OffsetRestrictionTracker.OffsetRange restriction, CommitLogReader reader, BlockingQueueLogObserver.CommitLogObserver observer)
org.apache.beam.sdk.transforms.DoFn.ProcessContinuation	process(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<OffsetRestrictionTracker.OffsetRange,Offset> tracker, org.apache.beam.sdk.transforms.DoFn.OutputReceiver<StreamElement> output, org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<?> watermarkEstimator, org.apache.beam.sdk.transforms.DoFn.BundleFinalizer finalizer)
void	setup()
(package private) void	splitRestriction(OffsetRestrictionTracker.OffsetRange restriction, org.apache.beam.sdk.transforms.DoFn.OutputReceiver<OffsetRestrictionTracker.OffsetRange> splits)
private void	startObserve(java.lang.String name, Partition partition, OffsetRestrictionTracker.OffsetRange restriction)
void	tearDown()

Methods inherited from class org.apache.beam.sdk.transforms.DoFn

getAllowedTimestampSkew, getInputTypeDescriptor, getOutputTypeDescriptor, populateDisplayData, prepareForProcessing

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

name

@Nullable
protected final java.lang.String name

position

protected final Position position

repositoryFactory

protected final RepositoryFactory repositoryFactory

readerFactory

protected final CommitLogReader.Factory<?> readerFactory

limit

protected final long limit

runningObserves

protected transient java.util.Map<java.lang.Integer,ObserveHandle> runningObserves

partitionToSeekedOffset

protected transient java.util.Map<java.lang.Integer,Offset> partitionToSeekedOffset

observers

protected transient java.util.Map<java.lang.Integer,BlockingQueueLogObserver.CommitLogObserver> observers

externalizableOffsets

private transient boolean externalizableOffsets

Constructor Detail

AbstractCommitLogReadFn

public AbstractCommitLogReadFn(@Nullable
                               java.lang.String name,
                               Position position,
                               long limit,
                               RepositoryFactory repositoryFactory,
                               CommitLogReader.Factory<?> readerFactory)

Method Detail

process

public org.apache.beam.sdk.transforms.DoFn.ProcessContinuation process(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<OffsetRestrictionTracker.OffsetRange,Offset> tracker,
                                                                       org.apache.beam.sdk.transforms.DoFn.OutputReceiver<StreamElement> output,
                                                                       org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator<?> watermarkEstimator,
                                                                       org.apache.beam.sdk.transforms.DoFn.BundleFinalizer finalizer)

closeHandleIfUnmatchingOffsets

private void closeHandleIfUnmatchingOffsets(org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker<OffsetRestrictionTracker.OffsetRange,Offset> tracker,
                                            Partition part,
                                            BlockingQueueLogObserver.CommitLogObserver observer)

closeHandle

protected void closeHandle(int part,
                           boolean nack)

startObserve

private void startObserve(@Nullable
                          java.lang.String name,
                          Partition partition,
                          OffsetRestrictionTracker.OffsetRange restriction)

observeBulkOffsets

abstract ObserveHandle observeBulkOffsets(OffsetRestrictionTracker.OffsetRange restriction,
                                          CommitLogReader reader,
                                          BlockingQueueLogObserver.CommitLogObserver observer)

observeBulkPartitions

abstract ObserveHandle observeBulkPartitions(@Nullable
                                             java.lang.String name,
                                             OffsetRestrictionTracker.OffsetRange restriction,
                                             CommitLogReader reader,
                                             BlockingQueueLogObserver.CommitLogObserver observer)

setup

public void setup()

tearDown

public void tearDown()

splitRestriction

void splitRestriction(OffsetRestrictionTracker.OffsetRange restriction,
                      org.apache.beam.sdk.transforms.DoFn.OutputReceiver<OffsetRestrictionTracker.OffsetRange> splits)

getRestrictionCoder

public org.apache.beam.sdk.coders.Coder<OffsetRestrictionTracker.OffsetRange> getRestrictionCoder()

newWatermarkEstimator

public org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.Manual newWatermarkEstimator(org.joda.time.Instant initialWatemark)

getInitialWatermarkEstimatorState

public org.joda.time.Instant getInitialWatermarkEstimatorState()

getWatermarkEstimatorStateCoder

public org.apache.beam.sdk.coders.Coder<org.joda.time.Instant> getWatermarkEstimatorStateCoder()