org.apache.nifi.processors.kafka.pubsub

Class ConsumerLease

java.lang.Object

org.apache.nifi.processors.kafka.pubsub.ConsumerLease

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.kafka.clients.consumer.ConsumerRebalanceListener

Direct Known Subclasses:

ConsumerPool.SimpleConsumerLease

public abstract class ConsumerLease
extends Object
implements Closeable, org.apache.kafka.clients.consumer.ConsumerRebalanceListener

This class represents a lease to access a Kafka Consumer object. The lease is intended to be obtained from a ConsumerPool. The lease is closeable to allow for the clean model of a try w/resources whereby non-exceptional cases mean the lease will be returned to the pool for future use by others. A given lease may only belong to a single thread a time.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	ConsumerLease.BundleInformation
private static class	ConsumerLease.BundleTracker

Field Summary

Fields

Modifier and Type	Field and Description
private String	bootstrapServers
private Map<ConsumerLease.BundleInformation,ConsumerLease.BundleTracker>	bundleMap
private byte[]	demarcatorBytes
private Charset	headerCharacterSet
private Pattern	headerNamePattern
private org.apache.kafka.clients.consumer.Consumer<byte[],byte[]>	kafkaConsumer
private String	keyEncoding
private boolean	lastPollEmpty
private long	leaseStartNanos
private ComponentLog	logger
private long	maxWaitMillis
private boolean	poisoned
private RecordReaderFactory	readerFactory
private String	securityProtocol
private int	totalMessages
private Map<org.apache.kafka.common.TopicPartition,org.apache.kafka.clients.consumer.OffsetAndMetadata>	uncommittedOffsetsMap
private RecordSetWriterFactory	writerFactory

Constructor Summary

Constructors

Constructor and Description
ConsumerLease(long maxWaitMillis, org.apache.kafka.clients.consumer.Consumer<byte[],byte[]> kafkaConsumer, byte[] demarcatorBytes, String keyEncoding, String securityProtocol, String bootstrapServers, RecordReaderFactory readerFactory, RecordSetWriterFactory writerFactory, ComponentLog logger, Charset headerCharacterSet, Pattern headerNamePattern)

Method Summary

Modifier and Type	Method and Description
void	close() Abstract method that is intended to be extended by the pool that created this ConsumerLease object.
private void	closeWriter(RecordSetWriter writer)
(package private) boolean	commit() Notifies Kafka to commit the offsets for the specified topic/partition pairs to the specified offsets w/the given metadata.
(package private) boolean	continuePolling() Indicates whether we should continue polling for data.
private static String	encodeKafkaKey(byte[] key, String encoding)
private Map<String,String>	getAttributes(org.apache.kafka.clients.consumer.ConsumerRecord<?,?> consumerRecord)
private Collection<FlowFile>	getBundles()
abstract ProcessSession	getProcessSession()
private void	handleParseFailure(org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> consumerRecord, ProcessSession session, Exception cause)
private void	handleParseFailure(org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> consumerRecord, ProcessSession session, Exception cause, String message)
(package private) boolean	isPoisoned()
void	onPartitionsAssigned(Collection<org.apache.kafka.common.TopicPartition> partitions) This will be called by Kafka when the rebalance has completed.
void	onPartitionsRevoked(Collection<org.apache.kafka.common.TopicPartition> partitions) Kafka will call this method whenever it is about to rebalance the consumers for the given partitions.
private void	poison() Indicates that the underlying session and consumer should be immediately considered invalid.
(package private) void	poll() Executes a poll on the underlying Kafka Consumer and creates any new flowfiles necessary or appends to existing ones if in demarcation mode.
private void	populateAttributes(ConsumerLease.BundleTracker tracker)
private boolean	processBundle(ConsumerLease.BundleTracker bundle)
private void	processRecords(org.apache.kafka.clients.consumer.ConsumerRecords<byte[],byte[]> records)
private void	resetInternalState() clears out internal state elements excluding session and consumer as those are managed by the pool itself
private void	rollback(org.apache.kafka.common.TopicPartition topicPartition)
void	wakeup() Trigger the consumer's wakeup() method.
private void	writeData(ProcessSession session, org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> record, org.apache.kafka.common.TopicPartition topicPartition)
private void	writeDemarcatedData(ProcessSession session, List<org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]>> records, org.apache.kafka.common.TopicPartition topicPartition)
private void	writeRecordData(ProcessSession session, List<org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]>> records, org.apache.kafka.common.TopicPartition topicPartition)
abstract void	yield()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

maxWaitMillis

private final long maxWaitMillis

kafkaConsumer

private final org.apache.kafka.clients.consumer.Consumer<byte[],byte[]> kafkaConsumer

logger

private final ComponentLog logger

demarcatorBytes

private final byte[] demarcatorBytes

keyEncoding

private final String keyEncoding

securityProtocol

private final String securityProtocol

bootstrapServers

private final String bootstrapServers

writerFactory

private final RecordSetWriterFactory writerFactory

readerFactory

private final RecordReaderFactory readerFactory

headerCharacterSet

private final Charset headerCharacterSet

headerNamePattern

private final Pattern headerNamePattern

poisoned

private boolean poisoned

bundleMap

private final Map<ConsumerLease.BundleInformation,ConsumerLease.BundleTracker> bundleMap

uncommittedOffsetsMap

private final Map<org.apache.kafka.common.TopicPartition,org.apache.kafka.clients.consumer.OffsetAndMetadata> uncommittedOffsetsMap

leaseStartNanos

private long leaseStartNanos

lastPollEmpty

private boolean lastPollEmpty

totalMessages

private int totalMessages

Constructor Detail

ConsumerLease

ConsumerLease(long maxWaitMillis,
              org.apache.kafka.clients.consumer.Consumer<byte[],byte[]> kafkaConsumer,
              byte[] demarcatorBytes,
              String keyEncoding,
              String securityProtocol,
              String bootstrapServers,
              RecordReaderFactory readerFactory,
              RecordSetWriterFactory writerFactory,
              ComponentLog logger,
              Charset headerCharacterSet,
              Pattern headerNamePattern)

Method Detail

resetInternalState

private void resetInternalState()

clears out internal state elements excluding session and consumer as those are managed by the pool itself

onPartitionsRevoked

public void onPartitionsRevoked(Collection<org.apache.kafka.common.TopicPartition> partitions)

Kafka will call this method whenever it is about to rebalance the consumers for the given partitions. We'll simply take this to mean that we need to quickly commit what we've got and will return the consumer to the pool. This method will be called during the poll() method call of this class and will be called by the same thread calling poll according to the Kafka API docs. After this method executes the session and kafka offsets are committed and this lease is closed.

Specified by:

onPartitionsRevoked in interface org.apache.kafka.clients.consumer.ConsumerRebalanceListener

Parameters:

partitions - partitions being reassigned

onPartitionsAssigned

public void onPartitionsAssigned(Collection<org.apache.kafka.common.TopicPartition> partitions)

This will be called by Kafka when the rebalance has completed. We don't need to do anything with this information other than optionally log it as by this point we've committed what we've got and moved on.

Specified by:

onPartitionsAssigned in interface org.apache.kafka.clients.consumer.ConsumerRebalanceListener

Parameters:

partitions - topic partition set being reassigned

poll

void poll()

Executes a poll on the underlying Kafka Consumer and creates any new flowfiles necessary or appends to existing ones if in demarcation mode.

commit

boolean commit()

Notifies Kafka to commit the offsets for the specified topic/partition pairs to the specified offsets w/the given metadata. This can offer higher performance than the other commitOffsets call as it allows the kafka client to collect more data from Kafka before committing the offsets. if false then we didn't do anything and should probably yield if true then we committed new data

continuePolling

boolean continuePolling()

Indicates whether we should continue polling for data. If we are not writing data with a demarcator then we're writing individual flow files per kafka message therefore we must be very mindful of memory usage for the flow file objects (not their content) being held in memory. The content of kafka messages will be written to the content repository immediately upon each poll call but we must still be mindful of how much memory can be used in each poll call. We will indicate that we should stop polling our last poll call produced no new results or if we've polling and processing data longer than the specified maximum polling time or if we have reached out specified max flow file limit or if a rebalance has been initiated for one of the partitions we're watching; otherwise true.

Returns:

true if should keep polling; false otherwise

poison

private void poison()

Indicates that the underlying session and consumer should be immediately considered invalid. Once closed the session will be rolled back and the pool should destroy the underlying consumer. This is useful if due to external reasons, such as the processor no longer being scheduled, this lease should be terminated immediately.

isPoisoned

boolean isPoisoned()

Returns:

true if this lease has been poisoned; false otherwise

wakeup

public void wakeup()

Trigger the consumer's wakeup() method.

close

public void close()

Abstract method that is intended to be extended by the pool that created this ConsumerLease object. It should ensure that the session given to create this session is rolled back and that the underlying kafka consumer is either returned to the pool for continued use or destroyed if this lease has been poisoned. It can only be called once. Calling it more than once can result in undefined and non threadsafe behavior.

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

getProcessSession

public abstract ProcessSession getProcessSession()

yield

public abstract void yield()

processRecords

private void processRecords(org.apache.kafka.clients.consumer.ConsumerRecords<byte[],byte[]> records)

encodeKafkaKey

private static String encodeKafkaKey(byte[] key,
                                     String encoding)

getBundles

private Collection<FlowFile> getBundles()
                                 throws IOException

Throws:

IOException

processBundle

private boolean processBundle(ConsumerLease.BundleTracker bundle)
                       throws IOException

Throws:

IOException

writeData

private void writeData(ProcessSession session,
                       org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> record,
                       org.apache.kafka.common.TopicPartition topicPartition)

writeDemarcatedData

private void writeDemarcatedData(ProcessSession session,
                                 List<org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]>> records,
                                 org.apache.kafka.common.TopicPartition topicPartition)

handleParseFailure

private void handleParseFailure(org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> consumerRecord,
                                ProcessSession session,
                                Exception cause)

handleParseFailure

private void handleParseFailure(org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]> consumerRecord,
                                ProcessSession session,
                                Exception cause,
                                String message)

getAttributes

private Map<String,String> getAttributes(org.apache.kafka.clients.consumer.ConsumerRecord<?,?> consumerRecord)

writeRecordData

private void writeRecordData(ProcessSession session,
                             List<org.apache.kafka.clients.consumer.ConsumerRecord<byte[],byte[]>> records,
                             org.apache.kafka.common.TopicPartition topicPartition)

closeWriter

private void closeWriter(RecordSetWriter writer)

rollback

private void rollback(org.apache.kafka.common.TopicPartition topicPartition)

populateAttributes

private void populateAttributes(ConsumerLease.BundleTracker tracker)