Package org.apache.nifi.processors.azure.eventhub

Class ConsumeAzureEventHub

java.lang.Object
org.apache.nifi.components.AbstractConfigurableComponent
org.apache.nifi.processor.AbstractSessionFactoryProcessor
org.apache.nifi.processors.azure.eventhub.ConsumeAzureEventHub
All Implemented Interfaces:
org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor, AzureEventHubComponent
@Tags({"azure","microsoft","cloud","eventhub","events","streaming","streams"}) @CapabilityDescription("Receives messages from Microsoft Azure Event Hubs with checkpointing to ensure consistent event processing. Checkpoint tracking avoids consuming a message multiple times and enables reliable resumption of processing in the event of intermittent network failures. Checkpoint tracking requires external storage and provides the preferred approach to consuming messages from Azure Event Hubs. In clustered environment, ConsumeAzureEventHub processor instances form a consumer group and the messages are distributed among the cluster nodes (each message is processed on one cluster node only).") @InputRequirement(INPUT_FORBIDDEN) @Stateful(scopes={LOCAL,CLUSTER}, description="Local state is used to store the client id. Cluster state is used to store partition ownership and checkpoint information when component state is configured as the checkpointing strategy.") @TriggerSerially @WritesAttribute(attribute="eventhub.enqueued.timestamp",description="The time (in milliseconds since epoch, UTC) at which the message was enqueued in the event hub") @WritesAttribute(attribute="eventhub.offset",description="The offset into the partition at which the message was stored") @WritesAttribute(attribute="eventhub.sequence",description="The sequence number associated with the message") @WritesAttribute(attribute="eventhub.name",description="The name of the event hub from which the message was pulled") @WritesAttribute(attribute="eventhub.partition",description="The name of the partition from which the message was pulled") @WritesAttribute(attribute="eventhub.property.*",description="The application properties of this message. IE: \'application\' would be \'eventhub.property.application\'") public class ConsumeAzureEventHub extends org.apache.nifi.processor.AbstractSessionFactoryProcessor implements AzureEventHubComponent

  • Field Summary

    Fields
    Modifier and Type
    Field
    Description
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    ACCESS_POLICY_NAME
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    BATCH_SIZE
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    CHECKPOINT_STRATEGY
     
    private String
    clientId
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    CONSUMER_GROUP
     
    private final Consumer<com.azure.messaging.eventhubs.models.ErrorContext>
    errorProcessor
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    EVENT_HUB_NAME
     
    protected final Consumer<com.azure.messaging.eventhubs.models.EventBatchContext>
    eventBatchProcessor
     
    private com.azure.messaging.eventhubs.EventProcessorClient
    eventProcessorClient
     
    private static final String
    FORMAT_STORAGE_CONNECTION_STRING_FOR_ACCOUNT_KEY
     
    private static final String
    FORMAT_STORAGE_CONNECTION_STRING_FOR_SAS_TOKEN
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    INITIAL_OFFSET
     
    (package private) static final org.apache.nifi.components.AllowableValue
    INITIAL_OFFSET_END_OF_STREAM
     
    (package private) static final org.apache.nifi.components.AllowableValue
    INITIAL_OFFSET_START_OF_STREAM
     
    private boolean
    isRecordReaderSet
     
    private boolean
    isRecordWriterSet
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    NAMESPACE
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    POLICY_PRIMARY_KEY
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    PREFETCH_COUNT
     
    private org.apache.nifi.processor.ProcessSessionFactory
    processSessionFactory
     
    private static final List<org.apache.nifi.components.PropertyDescriptor>
    PROPERTIES
     
    private RecordReaderFactory
    readerFactory
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    RECEIVE_TIMEOUT
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    RECORD_READER
     
    private static final Set<org.apache.nifi.processor.Relationship>
    RECORD_RELATIONSHIPS
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    RECORD_WRITER
     
    (package private) static final org.apache.nifi.processor.Relationship
    REL_PARSE_FAILURE
     
    (package private) static final org.apache.nifi.processor.Relationship
    REL_SUCCESS
     
    private static final Set<org.apache.nifi.processor.Relationship>
    RELATIONSHIPS
     
    private static final Pattern
    SAS_TOKEN_PATTERN
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    SERVICE_BUS_ENDPOINT
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    STORAGE_ACCOUNT_KEY
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    STORAGE_ACCOUNT_NAME
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    STORAGE_CONTAINER_NAME
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    STORAGE_SAS_TOKEN
     
    (package private) static final org.apache.nifi.components.PropertyDescriptor
    USE_MANAGED_IDENTITY
     
    private RecordSetWriterFactory
    writerFactory
     

    Fields inherited from interface org.apache.nifi.shared.azure.eventhubs.AzureEventHubComponent

    PROXY_CONFIGURATION_SERVICE, PROXY_SPECS, TRANSPORT_TYPE

  • Constructor Summary

    Constructors
    Constructor
    Description
    ConsumeAzureEventHub()
     

  • Method Summary

    Modifier and Type
    Method
    Description
    protected com.azure.messaging.eventhubs.EventProcessorClient
    createClient(org.apache.nifi.processor.ProcessContext context)
     
    private String
    createStorageConnectionString(org.apache.nifi.processor.ProcessContext context)
     
    protected Collection<org.apache.nifi.components.ValidationResult>
    customValidate(org.apache.nifi.components.ValidationContext validationContext)
     
    private Map<String,com.azure.messaging.eventhubs.models.EventPosition>
    getLegacyPartitionEventPosition(com.azure.storage.blob.BlobContainerAsyncClient blobContainerAsyncClient, String consumerGroup)
     
    Set<org.apache.nifi.processor.Relationship>
    getRelationships()
     
    protected List<org.apache.nifi.components.PropertyDescriptor>
    getSupportedPropertyDescriptors()
     
    protected String
    getTransitUri(com.azure.messaging.eventhubs.models.PartitionContext partitionContext)
     
    void
    migrateProperties(org.apache.nifi.migration.PropertyConfiguration config)
     
    void
    onPropertyModified(org.apache.nifi.components.PropertyDescriptor descriptor, String oldValue, String newValue)
     
    void
    onScheduled(org.apache.nifi.processor.ProcessContext context)
     
    void
    onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSessionFactory sessionFactory)
     
    private void
    putEventHubAttributes(Map<String,String> attributes, com.azure.messaging.eventhubs.models.PartitionContext partitionContext, com.azure.messaging.eventhubs.EventData eventData, com.azure.messaging.eventhubs.models.LastEnqueuedEventProperties lastEnqueuedEventProperties)
     
    void
    stopClient()
     
    private void
    transferTo(org.apache.nifi.processor.Relationship relationship, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch, com.azure.messaging.eventhubs.models.PartitionContext partitionContext, org.apache.nifi.flowfile.FlowFile flowFile)
     
    private void
    writeFlowFiles(com.azure.messaging.eventhubs.models.EventBatchContext eventBatchContext, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch)
     
    private void
    writeRecords(com.azure.messaging.eventhubs.models.EventBatchContext eventBatchContext, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch)
     

    Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

    getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

    Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

    equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, validate

    Methods inherited from class java.lang.Object

    clone, finalize, getClass, notify, notifyAll, wait, wait, wait

    Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

    getPropertyDescriptor, getPropertyDescriptors, validate

    Methods inherited from interface org.apache.nifi.processor.Processor

    isStateful, migrateRelationships

  • Field Details

    • SAS_TOKEN_PATTERN

      private static final Pattern SAS_TOKEN_PATTERN

    • FORMAT_STORAGE_CONNECTION_STRING_FOR_ACCOUNT_KEY

      private static final String FORMAT_STORAGE_CONNECTION_STRING_FOR_ACCOUNT_KEY
      See Also:

    • FORMAT_STORAGE_CONNECTION_STRING_FOR_SAS_TOKEN

      private static final String FORMAT_STORAGE_CONNECTION_STRING_FOR_SAS_TOKEN
      See Also:

    • NAMESPACE

      static final org.apache.nifi.components.PropertyDescriptor NAMESPACE

    • EVENT_HUB_NAME

      static final org.apache.nifi.components.PropertyDescriptor EVENT_HUB_NAME

    • SERVICE_BUS_ENDPOINT

      static final org.apache.nifi.components.PropertyDescriptor SERVICE_BUS_ENDPOINT

    • ACCESS_POLICY_NAME

      static final org.apache.nifi.components.PropertyDescriptor ACCESS_POLICY_NAME

    • POLICY_PRIMARY_KEY

      static final org.apache.nifi.components.PropertyDescriptor POLICY_PRIMARY_KEY

    • USE_MANAGED_IDENTITY

      static final org.apache.nifi.components.PropertyDescriptor USE_MANAGED_IDENTITY

    • CONSUMER_GROUP

      static final org.apache.nifi.components.PropertyDescriptor CONSUMER_GROUP

    • RECORD_READER

      static final org.apache.nifi.components.PropertyDescriptor RECORD_READER

    • RECORD_WRITER

      static final org.apache.nifi.components.PropertyDescriptor RECORD_WRITER

    • INITIAL_OFFSET_START_OF_STREAM

      static final org.apache.nifi.components.AllowableValue INITIAL_OFFSET_START_OF_STREAM

    • INITIAL_OFFSET_END_OF_STREAM

      static final org.apache.nifi.components.AllowableValue INITIAL_OFFSET_END_OF_STREAM

    • INITIAL_OFFSET

      static final org.apache.nifi.components.PropertyDescriptor INITIAL_OFFSET

    • PREFETCH_COUNT

      static final org.apache.nifi.components.PropertyDescriptor PREFETCH_COUNT

    • BATCH_SIZE

      static final org.apache.nifi.components.PropertyDescriptor BATCH_SIZE

    • RECEIVE_TIMEOUT

      static final org.apache.nifi.components.PropertyDescriptor RECEIVE_TIMEOUT

    • CHECKPOINT_STRATEGY

      static final org.apache.nifi.components.PropertyDescriptor CHECKPOINT_STRATEGY

    • STORAGE_ACCOUNT_NAME

      static final org.apache.nifi.components.PropertyDescriptor STORAGE_ACCOUNT_NAME

    • STORAGE_ACCOUNT_KEY

      static final org.apache.nifi.components.PropertyDescriptor STORAGE_ACCOUNT_KEY

    • STORAGE_SAS_TOKEN

      static final org.apache.nifi.components.PropertyDescriptor STORAGE_SAS_TOKEN

    • STORAGE_CONTAINER_NAME

      static final org.apache.nifi.components.PropertyDescriptor STORAGE_CONTAINER_NAME

    • REL_SUCCESS

      static final org.apache.nifi.processor.Relationship REL_SUCCESS

    • REL_PARSE_FAILURE

      static final org.apache.nifi.processor.Relationship REL_PARSE_FAILURE

    • RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RELATIONSHIPS

    • RECORD_RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RECORD_RELATIONSHIPS

    • PROPERTIES

      private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTIES

    • processSessionFactory

      private volatile org.apache.nifi.processor.ProcessSessionFactory processSessionFactory

    • eventProcessorClient

      private volatile com.azure.messaging.eventhubs.EventProcessorClient eventProcessorClient

    • readerFactory

      private volatile RecordReaderFactory readerFactory

    • writerFactory

      private volatile RecordSetWriterFactory writerFactory

    • isRecordReaderSet

      private volatile boolean isRecordReaderSet

    • isRecordWriterSet

      private volatile boolean isRecordWriterSet

    • clientId

      private volatile String clientId

    • eventBatchProcessor

      protected final Consumer<com.azure.messaging.eventhubs.models.EventBatchContext> eventBatchProcessor

    • errorProcessor

      private final Consumer<com.azure.messaging.eventhubs.models.ErrorContext> errorProcessor

  • Constructor Details

    • ConsumeAzureEventHub

      public ConsumeAzureEventHub()

  • Method Details

    • getSupportedPropertyDescriptors

      protected List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()
      Overrides:
      getSupportedPropertyDescriptors in class org.apache.nifi.components.AbstractConfigurableComponent

    • getRelationships

      public Set<org.apache.nifi.processor.Relationship> getRelationships()
      Specified by:
      getRelationships in interface org.apache.nifi.processor.Processor
      Overrides:
      getRelationships in class org.apache.nifi.processor.AbstractSessionFactoryProcessor

    • migrateProperties

      public void migrateProperties(org.apache.nifi.migration.PropertyConfiguration config)
      Specified by:
      migrateProperties in interface org.apache.nifi.processor.Processor

    • customValidate

      protected Collection<org.apache.nifi.components.ValidationResult> customValidate(org.apache.nifi.components.ValidationContext validationContext)
      Overrides:
      customValidate in class org.apache.nifi.components.AbstractConfigurableComponent

    • onPropertyModified

      public void onPropertyModified(org.apache.nifi.components.PropertyDescriptor descriptor, String oldValue, String newValue)
      Specified by:
      onPropertyModified in interface org.apache.nifi.components.ConfigurableComponent
      Overrides:
      onPropertyModified in class org.apache.nifi.components.AbstractConfigurableComponent

    • onScheduled

      @OnScheduled public void onScheduled(org.apache.nifi.processor.ProcessContext context) throws IOException
      Throws:
      IOException

    • onTrigger

      public void onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSessionFactory sessionFactory)
      Specified by:
      onTrigger in interface org.apache.nifi.processor.Processor

    • stopClient

      @OnStopped public void stopClient()

    • createClient

      protected com.azure.messaging.eventhubs.EventProcessorClient createClient(org.apache.nifi.processor.ProcessContext context)

    • getTransitUri

      protected String getTransitUri(com.azure.messaging.eventhubs.models.PartitionContext partitionContext)

    • putEventHubAttributes

      private void putEventHubAttributes(Map<String,String> attributes, com.azure.messaging.eventhubs.models.PartitionContext partitionContext, com.azure.messaging.eventhubs.EventData eventData, com.azure.messaging.eventhubs.models.LastEnqueuedEventProperties lastEnqueuedEventProperties)

    • writeFlowFiles

      private void writeFlowFiles(com.azure.messaging.eventhubs.models.EventBatchContext eventBatchContext, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch)

    • writeRecords

      private void writeRecords(com.azure.messaging.eventhubs.models.EventBatchContext eventBatchContext, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch) throws IOException
      Throws:
      IOException

    • transferTo

      private void transferTo(org.apache.nifi.processor.Relationship relationship, org.apache.nifi.processor.ProcessSession session, StopWatch stopWatch, com.azure.messaging.eventhubs.models.PartitionContext partitionContext, org.apache.nifi.flowfile.FlowFile flowFile)

    • createStorageConnectionString

      private String createStorageConnectionString(org.apache.nifi.processor.ProcessContext context)

    • getLegacyPartitionEventPosition

      private Map<String,com.azure.messaging.eventhubs.models.EventPosition> getLegacyPartitionEventPosition(com.azure.storage.blob.BlobContainerAsyncClient blobContainerAsyncClient, String consumerGroup)