@SideEffectFree @TriggerWhenEmpty @InputRequirement(value=INPUT_REQUIRED) @Tags(value={"merge","record","content","correlation","stream","event"}) @CapabilityDescription(value="This Processor merges together multiple record-oriented FlowFiles into a single FlowFile that contains all of the Records of the input FlowFiles. This Processor works by creating \'bins\' and then adding FlowFiles to these bins until they are full. Once a bin is full, all of the FlowFiles will be combined into a single output FlowFile, and that FlowFile will be routed to the \'merged\' Relationship. A bin will consist of potentially many \'like FlowFiles\'. In order for two FlowFiles to be considered \'like FlowFiles\', they must have the same Schema (as identified by the Record Reader) and, if the <Correlation Attribute Name> property is set, the same value for the specified attribute. See Processor Usage and Additional Details for more information.") @ReadsAttribute(attribute="fragment.identifier",description="Applicable only if the <Merge Strategy> property is set to Defragment. All FlowFiles with the same value for this attribute will be bundled together.") @ReadsAttribute(attribute="fragment.count",description="Applicable only if the <Merge Strategy> property is set to Defragment. This attribute must be present on all FlowFiles with the same value for the fragment.identifier attribute. All FlowFiles in the same bundle must have the same value for this attribute. The value of this attribute indicates how many FlowFiles should be expected in the given bundle.") @WritesAttribute(attribute="record.count",description="The merged FlowFile will have a \'record.count\' attribute indicating the number of records that were written to the FlowFile.") @WritesAttribute(attribute="mime.type",description="The MIME Type indicated by the Record Writer") @WritesAttribute(attribute="merge.count",description="The number of FlowFiles that were merged into this bundle") @WritesAttribute(attribute="merge.bin.age",description="The age of the bin, in milliseconds, when it was merged and output. Effectively this is the greatest amount of time that any FlowFile in this bundle remained waiting in this processor before it was output") @WritesAttribute(attribute="merge.uuid",description="UUID of the merged FlowFile that will be added to the original FlowFiles attributes") @WritesAttribute(attribute="<Attributes from Record Writer>",description="Any Attribute that the configured Record Writer returns will be added to the FlowFile.") @SeeAlso(value={MergeContent.class,SplitRecord.class,PartitionRecord.class}) public class MergeRecord extends AbstractSessionFactoryProcessor
Modifier and Type | Field and Description |
---|---|
private AtomicReference<RecordBinManager> |
binManager |
static PropertyDescriptor |
CORRELATION_ATTRIBUTE_NAME |
static String |
FRAGMENT_COUNT_ATTRIBUTE |
static String |
FRAGMENT_ID_ATTRIBUTE |
static String |
FRAGMENT_INDEX_ATTRIBUTE |
static PropertyDescriptor |
MAX_BIN_AGE |
static PropertyDescriptor |
MAX_BIN_COUNT |
static PropertyDescriptor |
MAX_RECORDS |
static PropertyDescriptor |
MAX_SIZE |
static String |
MERGE_BIN_AGE_ATTRIBUTE |
static String |
MERGE_COUNT_ATTRIBUTE |
static PropertyDescriptor |
MERGE_STRATEGY |
static AllowableValue |
MERGE_STRATEGY_BIN_PACK |
static AllowableValue |
MERGE_STRATEGY_DEFRAGMENT |
static String |
MERGE_UUID_ATTRIBUTE |
static PropertyDescriptor |
MIN_RECORDS |
static PropertyDescriptor |
MIN_SIZE |
static PropertyDescriptor |
RECORD_READER |
static PropertyDescriptor |
RECORD_WRITER |
static Relationship |
REL_FAILURE |
static Relationship |
REL_MERGED |
static Relationship |
REL_ORIGINAL |
Constructor and Description |
---|
MergeRecord() |
Modifier and Type | Method and Description |
---|---|
private void |
binFlowFile(ProcessContext context,
FlowFile flowFile,
ProcessSession session,
RecordBinManager binManager,
boolean block) |
protected Collection<ValidationResult> |
customValidate(ValidationContext validationContext) |
(package private) int |
getBinCount() |
protected String |
getGroupId(ProcessContext context,
FlowFile flowFile,
RecordSchema schema,
ProcessSession session) |
Set<Relationship> |
getRelationships() |
protected List<PropertyDescriptor> |
getSupportedPropertyDescriptors() |
void |
onTrigger(ProcessContext context,
ProcessSessionFactory sessionFactory) |
void |
resetState() |
getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue
equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate
clone, finalize, getClass, notify, notifyAll, wait, wait, wait
getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate
public static final String FRAGMENT_ID_ATTRIBUTE
public static final String FRAGMENT_INDEX_ATTRIBUTE
public static final String FRAGMENT_COUNT_ATTRIBUTE
public static final String MERGE_COUNT_ATTRIBUTE
public static final String MERGE_BIN_AGE_ATTRIBUTE
public static final String MERGE_UUID_ATTRIBUTE
public static final AllowableValue MERGE_STRATEGY_BIN_PACK
public static final AllowableValue MERGE_STRATEGY_DEFRAGMENT
public static final PropertyDescriptor RECORD_READER
public static final PropertyDescriptor RECORD_WRITER
public static final PropertyDescriptor MERGE_STRATEGY
public static final PropertyDescriptor CORRELATION_ATTRIBUTE_NAME
public static final PropertyDescriptor MIN_SIZE
public static final PropertyDescriptor MAX_SIZE
public static final PropertyDescriptor MIN_RECORDS
public static final PropertyDescriptor MAX_RECORDS
public static final PropertyDescriptor MAX_BIN_COUNT
public static final PropertyDescriptor MAX_BIN_AGE
public static final Relationship REL_MERGED
public static final Relationship REL_ORIGINAL
public static final Relationship REL_FAILURE
private final AtomicReference<RecordBinManager> binManager
protected List<PropertyDescriptor> getSupportedPropertyDescriptors()
getSupportedPropertyDescriptors
in class AbstractConfigurableComponent
public Set<Relationship> getRelationships()
getRelationships
in interface Processor
getRelationships
in class AbstractSessionFactoryProcessor
@OnStopped public final void resetState()
protected Collection<ValidationResult> customValidate(ValidationContext validationContext)
customValidate
in class AbstractConfigurableComponent
public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException
ProcessException
private void binFlowFile(ProcessContext context, FlowFile flowFile, ProcessSession session, RecordBinManager binManager, boolean block)
protected String getGroupId(ProcessContext context, FlowFile flowFile, RecordSchema schema, ProcessSession session)
int getBinCount()
Copyright © 2020 Apache NiFi Project. All rights reserved.