cz.o2.proxima.direct.pubsub

Class PubSubReader

java.lang.Object

cz.o2.proxima.storage.AbstractStorage

cz.o2.proxima.direct.pubsub.PubSubReader

All Implemented Interfaces:

CommitLogReader

@Stable
class PubSubReader
extends cz.o2.proxima.storage.AbstractStorage
implements CommitLogReader

A CommitLogReader for Google PubSub.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static interface	PubSubReader.PubSubConsumer
(package private) static class	PubSubReader.PubSubOffset
(package private) static class	PubSubReader.PubSubPartition

Nested classes/interfaces inherited from class cz.o2.proxima.storage.AbstractStorage

cz.o2.proxima.storage.AbstractStorage.SerializableAbstractStorage

Nested classes/interfaces inherited from interface cz.o2.proxima.direct.commitlog.CommitLogReader

CommitLogReader.Factory<T extends CommitLogReader>

Field Summary

Fields

Modifier and Type	Field and Description
private PubSubAccessor	accessor
private java.util.Map<java.lang.String,java.lang.Object>	cfg
private Context	context
private java.util.concurrent.ExecutorService	executor
private int	maxAckDeadline
private java.lang.String	project
private int	subscriptionAckDeadline
private boolean	subscriptionAutoCreate
private java.lang.String	topic
private PubSubWatermarkConfiguration	watermarkConfiguration

Constructor Summary

Constructors

Constructor and Description
PubSubReader(PubSubAccessor accessor, Context context)

Method Summary

Modifier and Type	Method and Description
private java.lang.String	asConsumerName(java.lang.String name)
CommitLogReader.Factory<?>	asFactory()
private ObserveHandle	consume(java.lang.String consumerName, PubSubReader.PubSubConsumer consumer, cz.o2.proxima.functional.UnaryFunction<java.lang.Throwable,java.lang.Boolean> errorHandler, java.lang.Runnable onInit, java.lang.Runnable onRestart, java.lang.Runnable onCancel, java.util.concurrent.atomic.AtomicLong committedWatermark)
private CommitLogObserver.OffsetCommitter	createBulkCommitter(java.lang.Object listLock, long confirmUntil, java.util.concurrent.atomic.AtomicLong globalOffset, java.util.concurrent.atomic.AtomicReference<java.util.List<com.google.cloud.pubsub.v1.AckReplyConsumer>> unconfirmed, cz.o2.proxima.time.WatermarkSupplier watermarkSupplier, java.util.concurrent.atomic.AtomicLong committedWatermark)
private com.google.cloud.pubsub.v1.MessageReceiver	createMessageReceiver(com.google.pubsub.v1.ProjectSubscriptionName subscription, java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.Subscriber> subscriber, java.util.concurrent.atomic.AtomicBoolean stopProcessing, PubSubReader.PubSubConsumer consumer, cz.o2.proxima.time.WatermarkEstimator watermarkEstimator, cz.o2.proxima.functional.UnaryFunction<java.lang.Throwable,java.lang.Boolean> errorHandler, java.lang.Runnable onRestart, java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.MessageReceiver> receiver)
private void	createSubscription(com.google.cloud.pubsub.v1.SubscriptionAdminClient client, com.google.pubsub.v1.ProjectSubscriptionName subscription)
(package private) cz.o2.proxima.time.WatermarkEstimator	createWatermarkEstimator(long minWatermark)
(package private) java.util.concurrent.ExecutorService	executor()
private void	failUnsupported()
private java.lang.String	findConsumerFromPartitions(java.lang.String name, java.util.Collection<cz.o2.proxima.storage.Partition> partitions)
java.util.List<cz.o2.proxima.storage.Partition>	getPartitions()
boolean	hasExternalizableOffsets()
(package private) com.google.cloud.pubsub.v1.Subscriber	newSubscriber(com.google.pubsub.v1.ProjectSubscriptionName subscription, com.google.cloud.pubsub.v1.MessageReceiver receiver)
ObserveHandle	observe(java.lang.String name, cz.o2.proxima.storage.commitlog.Position position, CommitLogObserver observer)
private ObserveHandle	observe(java.lang.String name, cz.o2.proxima.storage.commitlog.Position position, long minWatermark, CommitLogObserver observer)
ObserveHandle	observeBulk(java.lang.String name, cz.o2.proxima.storage.commitlog.Position position, boolean stopAtCurrent, CommitLogObserver observer) Observe PubSub in a bulk fashion.
private ObserveHandle	observeBulk(java.lang.String name, cz.o2.proxima.storage.commitlog.Position position, boolean stopAtCurrent, long minWatermark, CommitLogObserver observer)
ObserveHandle	observeBulkOffsets(java.util.Collection<Offset> offsets, boolean stopAtCurrent, CommitLogObserver observer)
ObserveHandle	observeBulkPartitions(java.lang.String name, java.util.Collection<cz.o2.proxima.storage.Partition> partitions, cz.o2.proxima.storage.commitlog.Position position, boolean stopAtCurrent, CommitLogObserver observer)
private ObserveHandle	observeBulkWithMinWatermark(java.lang.String name, cz.o2.proxima.storage.commitlog.Position position, boolean stopAtCurrent, long minWatermark, CommitLogObserver observer)
ObserveHandle	observePartitions(java.lang.String name, java.util.Collection<cz.o2.proxima.storage.Partition> partitions, cz.o2.proxima.storage.commitlog.Position position, boolean stopAtCurrent, CommitLogObserver observer)
(package private) com.google.cloud.pubsub.v1.Subscriber	stopAsync(java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.Subscriber> subscriber)
private void	validateNotStopAtCurrent(boolean stopAtCurrent)
private void	validatePosition(cz.o2.proxima.storage.commitlog.Position position)

Methods inherited from class cz.o2.proxima.storage.AbstractStorage

equals, getEntityDescriptor, getUri, hashCode

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface cz.o2.proxima.direct.commitlog.CommitLogReader

fetchOffsets, getOffsetExternalizer, getUri, observe, observeBulk, observeBulk, observeBulkOffsets, observeBulkPartitions, observeBulkPartitions, observeBulkPartitions, observePartitions, observePartitions, observePartitions, restoresSequentialIds

Field Detail

accessor

private final PubSubAccessor accessor

cfg

private final java.util.Map<java.lang.String,java.lang.Object> cfg

context

private final Context context

project

private final java.lang.String project

topic

private final java.lang.String topic

maxAckDeadline

private final int maxAckDeadline

subscriptionAckDeadline

private final int subscriptionAckDeadline

subscriptionAutoCreate

private final boolean subscriptionAutoCreate

watermarkConfiguration

private final PubSubWatermarkConfiguration watermarkConfiguration

executor

private transient java.util.concurrent.ExecutorService executor

Constructor Detail

PubSubReader

PubSubReader(PubSubAccessor accessor,
             Context context)

Method Detail

getPartitions

public java.util.List<cz.o2.proxima.storage.Partition> getPartitions()

Specified by:

getPartitions in interface CommitLogReader

observe

public ObserveHandle observe(@Nullable
                             java.lang.String name,
                             cz.o2.proxima.storage.commitlog.Position position,
                             CommitLogObserver observer)

Specified by:

observe in interface CommitLogReader

observe

private ObserveHandle observe(@Nullable
                              java.lang.String name,
                              cz.o2.proxima.storage.commitlog.Position position,
                              long minWatermark,
                              CommitLogObserver observer)

observePartitions

public ObserveHandle observePartitions(@Nullable
                                       java.lang.String name,
                                       java.util.Collection<cz.o2.proxima.storage.Partition> partitions,
                                       cz.o2.proxima.storage.commitlog.Position position,
                                       boolean stopAtCurrent,
                                       CommitLogObserver observer)

Specified by:

observePartitions in interface CommitLogReader

observeBulk

public ObserveHandle observeBulk(@Nullable
                                 java.lang.String name,
                                 cz.o2.proxima.storage.commitlog.Position position,
                                 boolean stopAtCurrent,
                                 CommitLogObserver observer)

Observe PubSub in a bulk fashion. Note that due to current PubSub implementation the bulk commit must happen before the ack timeout. If the message is not acknowledged before this timeout the message will be redelivered, which will result in duplicate messages.

Specified by:

observeBulk in interface CommitLogReader

Parameters:

name - name of the observer subscription

position - must be set to NEWEST

stopAtCurrent - throw UnsupportedOperationException when true

observer - the observer of data

Returns:

handle to interact with the observation thread

observeBulk

private ObserveHandle observeBulk(@Nullable
                                  java.lang.String name,
                                  cz.o2.proxima.storage.commitlog.Position position,
                                  boolean stopAtCurrent,
                                  long minWatermark,
                                  CommitLogObserver observer)

createBulkCommitter

private CommitLogObserver.OffsetCommitter createBulkCommitter(java.lang.Object listLock,
                                                              long confirmUntil,
                                                              java.util.concurrent.atomic.AtomicLong globalOffset,
                                                              java.util.concurrent.atomic.AtomicReference<java.util.List<com.google.cloud.pubsub.v1.AckReplyConsumer>> unconfirmed,
                                                              cz.o2.proxima.time.WatermarkSupplier watermarkSupplier,
                                                              java.util.concurrent.atomic.AtomicLong committedWatermark)

observeBulkPartitions

public ObserveHandle observeBulkPartitions(@Nullable
                                           java.lang.String name,
                                           java.util.Collection<cz.o2.proxima.storage.Partition> partitions,
                                           cz.o2.proxima.storage.commitlog.Position position,
                                           boolean stopAtCurrent,
                                           CommitLogObserver observer)

Specified by:

observeBulkPartitions in interface CommitLogReader

observeBulkWithMinWatermark

private ObserveHandle observeBulkWithMinWatermark(@Nullable
                                                  java.lang.String name,
                                                  cz.o2.proxima.storage.commitlog.Position position,
                                                  boolean stopAtCurrent,
                                                  long minWatermark,
                                                  CommitLogObserver observer)

observeBulkOffsets

public ObserveHandle observeBulkOffsets(java.util.Collection<Offset> offsets,
                                        boolean stopAtCurrent,
                                        CommitLogObserver observer)

Specified by:

observeBulkOffsets in interface CommitLogReader

newSubscriber

com.google.cloud.pubsub.v1.Subscriber newSubscriber(com.google.pubsub.v1.ProjectSubscriptionName subscription,
                                                    com.google.cloud.pubsub.v1.MessageReceiver receiver)

createWatermarkEstimator

cz.o2.proxima.time.WatermarkEstimator createWatermarkEstimator(long minWatermark)

createSubscription

private void createSubscription(com.google.cloud.pubsub.v1.SubscriptionAdminClient client,
                                com.google.pubsub.v1.ProjectSubscriptionName subscription)

validatePosition

private void validatePosition(cz.o2.proxima.storage.commitlog.Position position)

validateNotStopAtCurrent

private void validateNotStopAtCurrent(boolean stopAtCurrent)

failUnsupported

private void failUnsupported()

asConsumerName

private java.lang.String asConsumerName(java.lang.String name)

consume

private ObserveHandle consume(java.lang.String consumerName,
                              PubSubReader.PubSubConsumer consumer,
                              cz.o2.proxima.functional.UnaryFunction<java.lang.Throwable,java.lang.Boolean> errorHandler,
                              @Nullable
                              java.lang.Runnable onInit,
                              java.lang.Runnable onRestart,
                              java.lang.Runnable onCancel,
                              java.util.concurrent.atomic.AtomicLong committedWatermark)

createMessageReceiver

private com.google.cloud.pubsub.v1.MessageReceiver createMessageReceiver(com.google.pubsub.v1.ProjectSubscriptionName subscription,
                                                                         java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.Subscriber> subscriber,
                                                                         java.util.concurrent.atomic.AtomicBoolean stopProcessing,
                                                                         PubSubReader.PubSubConsumer consumer,
                                                                         cz.o2.proxima.time.WatermarkEstimator watermarkEstimator,
                                                                         cz.o2.proxima.functional.UnaryFunction<java.lang.Throwable,java.lang.Boolean> errorHandler,
                                                                         java.lang.Runnable onRestart,
                                                                         java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.MessageReceiver> receiver)

stopAsync

com.google.cloud.pubsub.v1.Subscriber stopAsync(java.util.concurrent.atomic.AtomicReference<com.google.cloud.pubsub.v1.Subscriber> subscriber)

executor

java.util.concurrent.ExecutorService executor()

hasExternalizableOffsets

public boolean hasExternalizableOffsets()

Specified by:

hasExternalizableOffsets in interface CommitLogReader

asFactory

public CommitLogReader.Factory<?> asFactory()

Specified by:

asFactory in interface CommitLogReader

findConsumerFromPartitions

private java.lang.String findConsumerFromPartitions(java.lang.String name,
                                                    java.util.Collection<cz.o2.proxima.storage.Partition> partitions)