Package com.networknt.kafka.consumer

Class KafkaConsumerManager

java.lang.Object

com.networknt.kafka.consumer.KafkaConsumerManager

public class KafkaConsumerManager
extends Object

Manages consumer instances by mapping instance IDs to consumer objects, processing read requests, and cleaning up when consumers disappear.

For read and commitOffsets tasks, it uses a ThreadPoolExecutor which spins up threads for handling read tasks. Since read tasks do not complete on the first run but rather call the AK consumer's poll() method continuously, we re-schedule them via a DelayQueue. A KafkaConsumerManager.ReadTaskSchedulerThread runs in a separate thread and re-submits the tasks to the executor.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static interface	KafkaConsumerManager.CommitCallback
static interface	KafkaConsumerManager.KafkaConsumerFactory

Field Summary

Fields

Modifier and Type	Field	Description
static com.networknt.client.Http2Client	client
static io.undertow.client.ClientConnection	connection

Constructor Summary

Constructors

Constructor	Description
KafkaConsumerManager(KafkaConsumerConfig config)

Method Summary

Modifier and Type	Method	Description
void	assign(String group, String instance, ConsumerAssignmentRequest assignmentRequest)
ConsumerAssignmentResponse	assignment(String group, String instance)
List<TopicPartitionOffset>	commitCurrentOffsets(String group, String instance)
Future	commitOffsets(String group, String instance, boolean async, ConsumerOffsetCommitRequest offsetCommitRequest, KafkaConsumerManager.CommitCallback callback)
ConsumerCommittedResponse	committed(String group, String instance, ConsumerCommittedRequest request)
String	createConsumer(String group, ConsumerInstanceConfig instanceConfig)	Creates a new consumer instance and returns its unique ID.
void	deleteConsumer(String group, String instance)
long	getBeginningOffset(String topic, int partition)	Returns the beginning offset of the topic partition.
io.undertow.client.ClientConnection	getConnection()
long	getEndOffset(String topic, int partition)	Returns the end offset of the topic partition.
KafkaConsumerState<?,?,?,?>	getExistingConsumerInstance(String group, String instance)
Optional<Long>	getOffsetForTime(String topic, int partition, Instant timestamp)	Returns the earliest offset whose timestamp is greater than or equal to the given timestamp in the topic partition, or empty if such offset does not exist.
org.apache.kafka.common.header.Headers	populateHeaders(RecordProcessedResult recordProcessedResult)
<KafkaKeyT,KafkaValueT,ClientKeyT,ClientValueT> void	readRecords(String group, String instance, Class<? extends KafkaConsumerState> consumerStateType, Duration timeout, long maxBytes, ConsumerReadCallback<ClientKeyT,ClientValueT> callback)
<ClientValueT,ClientKeyT> void	rollback(List<ConsumerRecord<ClientKeyT,ClientValueT>> records, String groupId, String instanceId)
void	rollbackExchangeDefinition(io.undertow.server.HttpServerExchange exchange, String groupId, String instanceId, List<String> topics, List<ConsumerRecord<Object,Object>> records)
void	seek(String group, String instance, ConsumerSeekRequest request)
<ClientValueT,ClientKeyT> List<ConsumerSeekRequest.PartitionOffset>	seekOffsetListUtility(List<ConsumerRecord<ClientKeyT,ClientValueT>> records)
void	seekToBeginning(String group, String instance, ConsumerSeekToRequest seekToRequest)
void	seekToEnd(String group, String instance, ConsumerSeekToRequest seekToRequest)
<ClientValueT,ClientKeyT> void	seekToParticularOffset(List<ConsumerRecord<ClientKeyT,ClientValueT>> records, String groupId, String instanceId)
void	shutdown()
void	subscribe(String group, String instance, ConsumerSubscriptionRecord subscription)
ConsumerSubscriptionResponse	subscription(String group, String instance)
void	successExchangeDefinition(io.undertow.server.HttpServerExchange exchange, String groupId, String instanceId, List<String> topics, List<ConsumerRecord<Object,Object>> records)
void	unsubscribe(String group, String instance)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

connection

public static io.undertow.client.ClientConnection connection

client

public static com.networknt.client.Http2Client client

Constructor Detail

KafkaConsumerManager

public KafkaConsumerManager(KafkaConsumerConfig config)

Method Detail

createConsumer

public String createConsumer(String group,
                             ConsumerInstanceConfig instanceConfig)

Creates a new consumer instance and returns its unique ID.

Parameters:

group - Name of the consumer group to join

instanceConfig - configuration parameters for the consumer

Returns:

Unique consumer instance ID

readRecords

public <KafkaKeyT,KafkaValueT,ClientKeyT,ClientValueT> void readRecords(String group,
                                                                                          String instance,
                                                                                          Class<? extends KafkaConsumerState> consumerStateType,
                                                                                          Duration timeout,
                                                                                          long maxBytes,
                                                                                          ConsumerReadCallback<ClientKeyT,ClientValueT> callback)

commitCurrentOffsets

public List<TopicPartitionOffset> commitCurrentOffsets(String group,
                                                       String instance)
                                                throws com.networknt.exception.FrameworkException

Throws:

com.networknt.exception.FrameworkException

commitOffsets

public Future commitOffsets(String group,
                            String instance,
                            boolean async,
                            ConsumerOffsetCommitRequest offsetCommitRequest,
                            KafkaConsumerManager.CommitCallback callback)

committed

public ConsumerCommittedResponse committed(String group,
                                           String instance,
                                           ConsumerCommittedRequest request)

getBeginningOffset

public long getBeginningOffset(String topic,
                               int partition)

Returns the beginning offset of the topic partition.

Parameters:

topic - the topic

partition - the partition

Returns:

the beginning offset

getEndOffset

public long getEndOffset(String topic,
                         int partition)

Returns the end offset of the topic partition.

Parameters:

topic - the topic

partition - the partition

Returns:

the offset

getOffsetForTime

public Optional<Long> getOffsetForTime(String topic,
                                       int partition,
                                       Instant timestamp)

Returns the earliest offset whose timestamp is greater than or equal to the given timestamp in the topic partition, or empty if such offset does not exist.

Parameters:

topic - the topic

partition - the partition

timestamp - the timestamp

Returns:

offset for timestamp

deleteConsumer

public void deleteConsumer(String group,
                           String instance)

subscribe

public void subscribe(String group,
                      String instance,
                      ConsumerSubscriptionRecord subscription)

unsubscribe

public void unsubscribe(String group,
                        String instance)

subscription

public ConsumerSubscriptionResponse subscription(String group,
                                                 String instance)

seekToBeginning

public void seekToBeginning(String group,
                            String instance,
                            ConsumerSeekToRequest seekToRequest)

seekToEnd

public void seekToEnd(String group,
                      String instance,
                      ConsumerSeekToRequest seekToRequest)

seek

public void seek(String group,
                 String instance,
                 ConsumerSeekRequest request)

assign

public void assign(String group,
                   String instance,
                   ConsumerAssignmentRequest assignmentRequest)

assignment

public ConsumerAssignmentResponse assignment(String group,
                                             String instance)

shutdown

public void shutdown()

getExistingConsumerInstance

public KafkaConsumerState<?,?,?,?> getExistingConsumerInstance(String group,
                                                                                 String instance)

rollback

public <ClientValueT,ClientKeyT> void rollback(List<ConsumerRecord<ClientKeyT,ClientValueT>> records,
                                                     String groupId,
                                                     String instanceId)

seekToParticularOffset

public <ClientValueT,ClientKeyT> void seekToParticularOffset(List<ConsumerRecord<ClientKeyT,ClientValueT>> records,
                                                                   String groupId,
                                                                   String instanceId)

seekOffsetListUtility

public <ClientValueT,ClientKeyT> List<ConsumerSeekRequest.PartitionOffset> seekOffsetListUtility(List<ConsumerRecord<ClientKeyT,ClientValueT>> records)

populateHeaders

public org.apache.kafka.common.header.Headers populateHeaders(RecordProcessedResult recordProcessedResult)

rollbackExchangeDefinition

public void rollbackExchangeDefinition(io.undertow.server.HttpServerExchange exchange,
                                       String groupId,
                                       String instanceId,
                                       List<String> topics,
                                       List<ConsumerRecord<Object,Object>> records)

successExchangeDefinition

public void successExchangeDefinition(io.undertow.server.HttpServerExchange exchange,
                                      String groupId,
                                      String instanceId,
                                      List<String> topics,
                                      List<ConsumerRecord<Object,Object>> records)

getConnection

public io.undertow.client.ClientConnection getConnection()