cz.o2.proxima.utils.zookeeper

Class ZKGlobalWatermarkTracker

java.lang.Object

cz.o2.proxima.utils.zookeeper.ZKGlobalWatermarkTracker

All Implemented Interfaces:

TimeProvider, GlobalWatermarkTracker, WatermarkSupplier, java.io.Closeable, java.io.Serializable, java.lang.AutoCloseable

public class ZKGlobalWatermarkTracker
extends java.lang.Object
implements GlobalWatermarkTracker

A GlobalWatermarkTracker that stores global information in Apache Zookeeper.

See Also:

Serialized Form

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
(package private) static class	ZKGlobalWatermarkTracker.WatermarkWithUpdate

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	CFG_MAX_ACCEPTABLE_UPDATE_AGE_MS
static java.lang.String	CFG_NAME
static java.lang.String	CFG_TIME_PROVIDER
private org.apache.zookeeper.ZooKeeper	client
private boolean	closed
private java.util.concurrent.atomic.AtomicLong	globalWatermark
private static long	MAX_WATERMARK
private long	maxAcceptableUpdateMs
private boolean	parentCreated
private org.apache.zookeeper.CreateMode	parentCreateMode
(package private) java.lang.String	parentNode
private java.util.Map<java.lang.String,ZKGlobalWatermarkTracker.WatermarkWithUpdate>	partialWatermarks
(package private) java.util.Map<java.lang.String,java.lang.Integer>	pathToVersion
private static long	serialVersionUID
private int	sessionTimeout
(package private) TimeProvider	timeProvider
private java.lang.String	trackerName
static java.lang.String	ZK_SESSION_TIMEOUT
static java.lang.String	ZK_URI
(package private) java.lang.String	zkConnectString

Constructor Summary

Constructors

Constructor and Description
ZKGlobalWatermarkTracker()

Method Summary

Modifier and Type	Method and Description
private org.apache.zookeeper.ZooKeeper	client()
void	close()
(package private) org.apache.zookeeper.ZooKeeper	createNewZooKeeper()
private void	createNodeIfNotExists(java.lang.String node)
private void	createNodeIntoFuture(java.lang.String name, byte[] bytes, java.util.concurrent.CompletableFuture<java.lang.Void> res)
(package private) void	createParentIfNotExists()
private void	createWatchForChildren(java.lang.String node)
private void	deleteNodeToFuture(java.lang.String name, java.util.concurrent.CompletableFuture<java.lang.Void> res)
private java.util.concurrent.CompletableFuture<java.lang.Void>	deletePartialWatermark(java.lang.String name)
(package private) void	disconnect()
(package private) int	forceUpdateVersion(java.lang.String path, int version)
(package private) static ZKGlobalWatermarkTracker.WatermarkWithUpdate	fromPayload(byte[] bytes)
long	getGlobalWatermark(java.lang.String processName, long currentWatermark)
private long	getMaxAcceptableUpdateAge(java.util.Map<java.lang.String,java.lang.Object> cfg)
java.lang.String	getName()
private java.lang.String	getNodeFromName(java.lang.String name)
(package private) static java.lang.String	getNodeName(java.lang.String path)
private java.util.concurrent.CompletableFuture<java.lang.Integer>	getNodeVersion(java.lang.String path)
private void	getNodeVersionToFuture(java.lang.String path, java.util.concurrent.CompletableFuture<java.lang.Integer> res)
(package private) java.lang.String	getParentNode()
private int	getSessionTimeout(java.util.Map<java.lang.String,java.lang.Object> cfg)
private TimeProvider	getTimeProvider(java.util.Map<java.lang.String,java.lang.Object> cfg)
private java.lang.String	getTrackerName(java.util.Map<java.lang.String,java.lang.Object> cfg)
(package private) org.apache.zookeeper.Watcher	getWatcher(java.util.concurrent.CountDownLatch connectLatch)
private java.net.URI	getZkUri(java.util.Map<java.lang.String,java.lang.Object> cfg)
private void	handleBadVersion(java.lang.String path, java.util.concurrent.CompletableFuture<java.lang.Void> res, java.lang.Runnable onSuccess)
private void	handleConnectionLoss(java.lang.Runnable retry)
private void	handleError(java.lang.Throwable err, java.lang.String logString, java.util.concurrent.CompletableFuture<java.lang.Void> future)
private void	handleNoParentNode(java.lang.String name, byte[] bytes, java.util.concurrent.CompletableFuture<java.lang.Void> res)
private void	handleWatchOnChildNode(java.lang.String path, boolean isDelete)
private void	handleWatchOnParentNode()
private void	init()
void	initWatermarks(java.util.Map<java.lang.String,java.lang.Long> initialWatermarks)
(package private) int	maybeUpdateVersion(java.lang.String path, int version)
private java.util.concurrent.CompletableFuture<java.lang.Void>	persistPartialWatermark(java.lang.String name, long watermark)
private void	persistPartialWatermarkIntoFuture(java.lang.String name, byte[] bytes, java.util.concurrent.CompletableFuture<java.lang.Void> res)
protected java.lang.Object	readResolve()
private void	setNodeDataToFuture(java.lang.String name, byte[] bytes, java.util.concurrent.CompletableFuture<java.lang.Void> res)
void	setup(java.util.Map<java.lang.String,java.lang.Object> cfg)
private int	setUpdateVersion(java.lang.String path, int version, boolean forceOverwrite)
private int	setUpdateVersion(java.lang.String path, int version, boolean forceOverwrite, Consumer<java.lang.Integer> oldVersionConsumer)
(package private) static byte[]	toPayload(long watermark, long updateTimestamp)
java.lang.String	toString()
java.util.concurrent.CompletableFuture<java.lang.Void>	update(java.lang.String processName, long currentWatermark)
private void	updatePartialWatermark(java.lang.String name, ZKGlobalWatermarkTracker.WatermarkWithUpdate update)
(package private) int	updateVersion(java.lang.String path)
private void	watchParentNode(org.apache.zookeeper.WatchedEvent watchedEvent)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface cz.o2.proxima.storage.watermark.GlobalWatermarkTracker

finished, getWatermark

Methods inherited from interface cz.o2.proxima.time.WatermarkSupplier

getCurrentTime

Methods inherited from interface cz.o2.proxima.functional.TimeProvider

processingTime

Field Detail

serialVersionUID

private static final long serialVersionUID

See Also:

Constant Field Values

MAX_WATERMARK

private static final long MAX_WATERMARK

See Also:

Constant Field Values

CFG_NAME

public static final java.lang.String CFG_NAME

See Also:

Constant Field Values

ZK_URI

public static final java.lang.String ZK_URI

See Also:

Constant Field Values

ZK_SESSION_TIMEOUT

public static final java.lang.String ZK_SESSION_TIMEOUT

See Also:

Constant Field Values

CFG_TIME_PROVIDER

public static final java.lang.String CFG_TIME_PROVIDER

See Also:

Constant Field Values

CFG_MAX_ACCEPTABLE_UPDATE_AGE_MS

public static final java.lang.String CFG_MAX_ACCEPTABLE_UPDATE_AGE_MS

See Also:

Constant Field Values

timeProvider

TimeProvider timeProvider

zkConnectString

java.lang.String zkConnectString

parentNode

java.lang.String parentNode

trackerName

private java.lang.String trackerName

sessionTimeout

private int sessionTimeout

maxAcceptableUpdateMs

private long maxAcceptableUpdateMs

client

private transient volatile org.apache.zookeeper.ZooKeeper client

partialWatermarks

private transient java.util.Map<java.lang.String,ZKGlobalWatermarkTracker.WatermarkWithUpdate> partialWatermarks

closed

private transient volatile boolean closed

globalWatermark

private transient java.util.concurrent.atomic.AtomicLong globalWatermark

parentCreateMode

private transient volatile org.apache.zookeeper.CreateMode parentCreateMode

parentCreated

private transient volatile boolean parentCreated

pathToVersion

transient java.util.Map<java.lang.String,java.lang.Integer> pathToVersion

Constructor Detail

ZKGlobalWatermarkTracker

public ZKGlobalWatermarkTracker()

Method Detail

init

private void init()

getName

public java.lang.String getName()

Specified by:

getName in interface GlobalWatermarkTracker

setup

public void setup(java.util.Map<java.lang.String,java.lang.Object> cfg)

Specified by:

setup in interface GlobalWatermarkTracker

getMaxAcceptableUpdateAge

private long getMaxAcceptableUpdateAge(java.util.Map<java.lang.String,java.lang.Object> cfg)

getTimeProvider

private TimeProvider getTimeProvider(java.util.Map<java.lang.String,java.lang.Object> cfg)

getTrackerName

@Nonnull
private java.lang.String getTrackerName(java.util.Map<java.lang.String,java.lang.Object> cfg)

getSessionTimeout

private int getSessionTimeout(java.util.Map<java.lang.String,java.lang.Object> cfg)

getZkUri

private java.net.URI getZkUri(java.util.Map<java.lang.String,java.lang.Object> cfg)

initWatermarks

public void initWatermarks(java.util.Map<java.lang.String,java.lang.Long> initialWatermarks)

Specified by:

initWatermarks in interface GlobalWatermarkTracker

update

public java.util.concurrent.CompletableFuture<java.lang.Void> update(java.lang.String processName,
                                                                     long currentWatermark)

Specified by:

update in interface GlobalWatermarkTracker

getGlobalWatermark

public long getGlobalWatermark(@Nullable
                               java.lang.String processName,
                               long currentWatermark)

Specified by:

getGlobalWatermark in interface GlobalWatermarkTracker

getNodeName

static java.lang.String getNodeName(@Nonnull
                                    java.lang.String path)

close

public void close()

Specified by:

close in interface java.io.Closeable

Specified by:

close in interface java.lang.AutoCloseable

disconnect

void disconnect()

toString

public java.lang.String toString()

Overrides:

toString in class java.lang.Object

persistPartialWatermark

private java.util.concurrent.CompletableFuture<java.lang.Void> persistPartialWatermark(java.lang.String name,
                                                                                       long watermark)

persistPartialWatermarkIntoFuture

private void persistPartialWatermarkIntoFuture(java.lang.String name,
                                               byte[] bytes,
                                               java.util.concurrent.CompletableFuture<java.lang.Void> res)

handleError

private void handleError(java.lang.Throwable err,
                         java.lang.String logString,
                         java.util.concurrent.CompletableFuture<java.lang.Void> future)

handleNoParentNode

private void handleNoParentNode(java.lang.String name,
                                byte[] bytes,
                                java.util.concurrent.CompletableFuture<java.lang.Void> res)

deletePartialWatermark

private java.util.concurrent.CompletableFuture<java.lang.Void> deletePartialWatermark(java.lang.String name)

deleteNodeToFuture

private void deleteNodeToFuture(java.lang.String name,
                                java.util.concurrent.CompletableFuture<java.lang.Void> res)

getNodeFromName

private java.lang.String getNodeFromName(java.lang.String name)

setNodeDataToFuture

private void setNodeDataToFuture(java.lang.String name,
                                 byte[] bytes,
                                 java.util.concurrent.CompletableFuture<java.lang.Void> res)

createNodeIntoFuture

private void createNodeIntoFuture(java.lang.String name,
                                  byte[] bytes,
                                  java.util.concurrent.CompletableFuture<java.lang.Void> res)

getNodeVersion

private java.util.concurrent.CompletableFuture<java.lang.Integer> getNodeVersion(java.lang.String path)

getNodeVersionToFuture

private void getNodeVersionToFuture(java.lang.String path,
                                    java.util.concurrent.CompletableFuture<java.lang.Integer> res)

handleConnectionLoss

private void handleConnectionLoss(java.lang.Runnable retry)

handleBadVersion

private void handleBadVersion(java.lang.String path,
                              java.util.concurrent.CompletableFuture<java.lang.Void> res,
                              java.lang.Runnable onSuccess)

createParentIfNotExists

void createParentIfNotExists()
                      throws java.lang.InterruptedException

Throws:

java.lang.InterruptedException

createWatchForChildren

private void createWatchForChildren(java.lang.String node)
                             throws java.lang.InterruptedException

Throws:

java.lang.InterruptedException

createNodeIfNotExists

private void createNodeIfNotExists(java.lang.String node)
                            throws java.lang.InterruptedException,
                                   org.apache.zookeeper.KeeperException

Throws:

java.lang.InterruptedException

org.apache.zookeeper.KeeperException

toPayload

static byte[] toPayload(long watermark,
                        long updateTimestamp)

fromPayload

static ZKGlobalWatermarkTracker.WatermarkWithUpdate fromPayload(byte[] bytes)

getParentNode

java.lang.String getParentNode()

updatePartialWatermark

private void updatePartialWatermark(java.lang.String name,
                                    ZKGlobalWatermarkTracker.WatermarkWithUpdate update)

client

private org.apache.zookeeper.ZooKeeper client()

createNewZooKeeper

org.apache.zookeeper.ZooKeeper createNewZooKeeper()

getWatcher

org.apache.zookeeper.Watcher getWatcher(java.util.concurrent.CountDownLatch connectLatch)

watchParentNode

private void watchParentNode(org.apache.zookeeper.WatchedEvent watchedEvent)

handleWatchOnParentNode

private void handleWatchOnParentNode()

handleWatchOnChildNode

private void handleWatchOnChildNode(java.lang.String path,
                                    boolean isDelete)

updateVersion

int updateVersion(java.lang.String path)

maybeUpdateVersion

int maybeUpdateVersion(java.lang.String path,
                       int version)

forceUpdateVersion

int forceUpdateVersion(java.lang.String path,
                       int version)

setUpdateVersion

private int setUpdateVersion(java.lang.String path,
                             int version,
                             boolean forceOverwrite)

setUpdateVersion

private int setUpdateVersion(java.lang.String path,
                             int version,
                             boolean forceOverwrite,
                             Consumer<java.lang.Integer> oldVersionConsumer)

readResolve

protected java.lang.Object readResolve()