Package org.apache.nifi.processors.azure.storage

Class ListAzureDataLakeStorage

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processor.util.list.AbstractListProcessor<ADLSFileInfo>

org.apache.nifi.processors.azure.storage.AbstractListAzureProcessor<ADLSFileInfo>

org.apache.nifi.processors.azure.storage.ListAzureDataLakeStorage

All Implemented Interfaces:

org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor, org.apache.nifi.processor.VerifiableProcessor

@PrimaryNodeOnly
@TriggerSerially
@Tags({"azure","microsoft","cloud","storage","adlsgen2","datalake"})
@SeeAlso({PutAzureDataLakeStorage.class,DeleteAzureDataLakeStorage.class,FetchAzureDataLakeStorage.class})
@CapabilityDescription("Lists directory in an Azure Data Lake Storage Gen 2 filesystem")
@WritesAttribute(attribute="azure.filesystem",description="The name of the Azure File System") @WritesAttribute(attribute="azure.filePath",description="The full path of the Azure File") @WritesAttribute(attribute="azure.directory",description="The name of the Azure Directory") @WritesAttribute(attribute="azure.filename",description="The name of the Azure File") @WritesAttribute(attribute="azure.length",description="The length of the Azure File") @WritesAttribute(attribute="azure.lastModified",description="The last modification time of the Azure File") @WritesAttribute(attribute="azure.etag",description="The ETag of the Azure File")
@InputRequirement(INPUT_FORBIDDEN)
@Stateful(scopes=CLUSTER,
          description="After performing a listing of files, the timestamp of the newest file is stored. This allows the Processor to list only files that have been added or modified after this date the next time that the Processor is run. State is stored across the cluster so that this Processor can be run on Primary Node only and if a new Primary Node is selected, the new node can pick up where the previous node left off, without duplicating the data.")
@DefaultSchedule(strategy=TIMER_DRIVEN,
                 period="1 min")
public class ListAzureDataLakeStorage
extends AbstractListAzureProcessor<ADLSFileInfo>

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

AbstractListProcessor.ListingMode

Field Summary

Fields

Modifier and Type	Field	Description
private DataLakeServiceClientFactory	clientFactory
static final org.apache.nifi.components.PropertyDescriptor	FILE_FILTER
private Pattern	filePattern
static final org.apache.nifi.components.PropertyDescriptor	INCLUDE_TEMPORARY_FILES
private static final Set<org.apache.nifi.components.PropertyDescriptor>	LISTING_RESET_PROPERTIES
static final org.apache.nifi.components.PropertyDescriptor	PATH_FILTER
private Pattern	pathPattern
private static final List<org.apache.nifi.components.PropertyDescriptor>	PROPERTIES
static final org.apache.nifi.components.PropertyDescriptor	RECURSE_SUBDIRECTORIES

Fields inherited from class org.apache.nifi.processors.azure.storage.AbstractListAzureProcessor

MAX_AGE, MAX_SIZE, MIN_AGE, MIN_SIZE

Fields inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

BY_ENTITIES, BY_TIME_WINDOW, BY_TIMESTAMPS, LISTING_LAG_MILLIS, LISTING_STRATEGY, NO_TRACKING, PRECISION_AUTO_DETECT, PRECISION_MILLIS, PRECISION_MINUTES, PRECISION_SECONDS, RECORD_WRITER, REL_SUCCESS, TARGET_SYSTEM_TIMESTAMP_PRECISION

Constructor Summary

Constructors

Constructor	Description
ListAzureDataLakeStorage()

Method Summary

Modifier and Type	Method	Description
protected Integer	countUnfilteredListing(org.apache.nifi.processor.ProcessContext context)
protected Map<String,String>	createAttributes(ADLSFileInfo fileInfo, org.apache.nifi.processor.ProcessContext context)
protected void	customValidate(org.apache.nifi.components.ValidationContext context, Collection<org.apache.nifi.components.ValidationResult> results)
protected String	getDefaultTimePrecision()
protected String	getListingContainerName(org.apache.nifi.processor.ProcessContext context)
protected String	getPath(org.apache.nifi.processor.ProcessContext context)
private Pattern	getPattern(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.components.PropertyDescriptor filterDescriptor)
protected RecordSchema	getRecordSchema()
protected org.apache.nifi.components.state.Scope	getStateScope(org.apache.nifi.context.PropertyContext context)
protected List<org.apache.nifi.components.PropertyDescriptor>	getSupportedPropertyDescriptors()
protected boolean	isListingResetNecessary(org.apache.nifi.components.PropertyDescriptor property)
void	onScheduled(org.apache.nifi.processor.ProcessContext context)
void	onStopped()
protected List<ADLSFileInfo>	performListing(org.apache.nifi.processor.ProcessContext context, Long minTimestamp, AbstractListProcessor.ListingMode listingMode)
private List<ADLSFileInfo>	performListing(org.apache.nifi.processor.ProcessContext context, Long minTimestamp, AbstractListProcessor.ListingMode listingMode, boolean applyFilters)

Methods inherited from class org.apache.nifi.processors.azure.storage.AbstractListAzureProcessor

isFileInfoMatchesWithAgeAndSize

Methods inherited from class org.apache.nifi.processor.util.list.AbstractListProcessor

createListedEntityTracker, customValidate, getCurrentNanoTime, getCurrentTime, getKey, getRelationships, initListedEntityTracker, listByNoTracking, listByTimeWindow, listByTrackingTimestamps, migrateProperties, onPrimaryNodeChange, onPropertyModified, onTrigger, updateState, verify

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, validate

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful, migrateRelationships

Field Details

RECURSE_SUBDIRECTORIES

public static final org.apache.nifi.components.PropertyDescriptor RECURSE_SUBDIRECTORIES

FILE_FILTER

public static final org.apache.nifi.components.PropertyDescriptor FILE_FILTER

PATH_FILTER

public static final org.apache.nifi.components.PropertyDescriptor PATH_FILTER

INCLUDE_TEMPORARY_FILES

public static final org.apache.nifi.components.PropertyDescriptor INCLUDE_TEMPORARY_FILES

PROPERTIES

private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTIES

LISTING_RESET_PROPERTIES

private static final Set<org.apache.nifi.components.PropertyDescriptor> LISTING_RESET_PROPERTIES

filePattern

private volatile Pattern filePattern

pathPattern

private volatile Pattern pathPattern

clientFactory

private volatile DataLakeServiceClientFactory clientFactory

Constructor Details

ListAzureDataLakeStorage

public ListAzureDataLakeStorage()

Method Details

getSupportedPropertyDescriptors

protected List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class org.apache.nifi.components.AbstractConfigurableComponent

onScheduled

@OnScheduled
public void onScheduled(org.apache.nifi.processor.ProcessContext context)

onStopped

@OnStopped
public void onStopped()

customValidate

protected void customValidate(org.apache.nifi.components.ValidationContext context,
 Collection<org.apache.nifi.components.ValidationResult> results)

Overrides:

customValidate in class AbstractListProcessor<ADLSFileInfo>

getRecordSchema

protected RecordSchema getRecordSchema()

Specified by:

getRecordSchema in class AbstractListProcessor<ADLSFileInfo>

getStateScope

protected org.apache.nifi.components.state.Scope getStateScope(org.apache.nifi.context.PropertyContext context)

Specified by:

getStateScope in class AbstractListProcessor<ADLSFileInfo>

getDefaultTimePrecision

protected String getDefaultTimePrecision()

Overrides:

getDefaultTimePrecision in class AbstractListProcessor<ADLSFileInfo>

isListingResetNecessary

protected boolean isListingResetNecessary(org.apache.nifi.components.PropertyDescriptor property)

Specified by:

isListingResetNecessary in class AbstractListProcessor<ADLSFileInfo>

getPath

protected String getPath(org.apache.nifi.processor.ProcessContext context)

Specified by:

getPath in class AbstractListProcessor<ADLSFileInfo>

performListing

protected List<ADLSFileInfo> performListing(org.apache.nifi.processor.ProcessContext context,
 Long minTimestamp,
 AbstractListProcessor.ListingMode listingMode)
                                     throws IOException

Specified by:

performListing in class AbstractListProcessor<ADLSFileInfo>

Throws:

IOException

countUnfilteredListing

protected Integer countUnfilteredListing(org.apache.nifi.processor.ProcessContext context)
                                  throws IOException

Specified by:

countUnfilteredListing in class AbstractListProcessor<ADLSFileInfo>

Throws:

IOException

getListingContainerName

protected String getListingContainerName(org.apache.nifi.processor.ProcessContext context)

Specified by:

getListingContainerName in class AbstractListProcessor<ADLSFileInfo>

createAttributes

protected Map<String,String> createAttributes(ADLSFileInfo fileInfo,
 org.apache.nifi.processor.ProcessContext context)

Specified by:

createAttributes in class AbstractListProcessor<ADLSFileInfo>

performListing

private List<ADLSFileInfo> performListing(org.apache.nifi.processor.ProcessContext context,
 Long minTimestamp,
 AbstractListProcessor.ListingMode listingMode,
 boolean applyFilters)
                                   throws IOException

Throws:

IOException

getPattern

private Pattern getPattern(org.apache.nifi.processor.ProcessContext context,
 org.apache.nifi.components.PropertyDescriptor filterDescriptor)