From d3df7c683e835aa981fb0abd1fb0d34cdde30fc6 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Tue, 19 Sep 2023 17:31:43 +0200 Subject: [PATCH 1/5] MINIFICPP-2221 Auto-generate CONTROLLERS.md Also: * simplify the auto-generation of PROCESSORS.md: don't create individual files for each processor * add missing controller services to CONTROLLERS.md * move some descriptions from CONTROLLERS.md to the Description of the controller services * fix some Visual Studio warnings and a bug in formatAllowedValues(). --- CONTROLLERS.md | 297 +++++++++++++++--- PROCESSORS.md | 244 +++++++------- .../AWSCredentialsService.h | 4 +- .../AzureStorageCredentialsService.h | 3 +- .../GCPCredentialsControllerService.h | 3 +- .../include/controllers/SSLContextService.h | 10 +- libminifi/include/core/PropertyDefinition.h | 4 +- libminifi/include/core/PropertyType.h | 16 +- minifi_main/AgentDocs.cpp | 234 +++++++------- minifi_main/AgentDocs.h | 9 +- minifi_main/MiNiFiMain.cpp | 28 +- 11 files changed, 525 insertions(+), 327 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 0939a8846f..3468929deb 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -1,46 +1,53 @@ -# Controller Services - ## Table of Contents - [AWSCredentialsService](#AWSCredentialsService) - [AzureStorageCredentialsService](#AzureStorageCredentialsService) -- [GCPCredentialsControllerService](#GCPCredentialsControllerService) - [ElasticsearchCredentialsControllerService](#ElasticsearchCredentialsControllerService) -- [KubernetesControllerService](#kubernetesControllerService) +- [ExecuteJavaControllerService](#ExecuteJavaControllerService) +- [GCPCredentialsControllerService](#GCPCredentialsControllerService) +- [JavaControllerService](#JavaControllerService) +- [KubernetesControllerService](#KubernetesControllerService) +- [LinuxPowerManagerService](#LinuxPowerManagerService) +- [NetworkPrioritizerService](#NetworkPrioritizerService) +- [ODBCService](#ODBCService) +- [PersistentMapStateStorage](#PersistentMapStateStorage) +- [RocksDbStateStorage](#RocksDbStateStorage) +- [SmbConnectionControllerService](#SmbConnectionControllerService) +- [SSLContextService](#SSLContextService) +- [UpdatePolicyControllerService](#UpdatePolicyControllerService) +- [VolatileMapStateStorage](#VolatileMapStateStorage) + ## AWSCredentialsService ### Description -Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple -AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single -controller service so that AWS credentials can be managed and controlled in a central location. +Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single controller service so that AWS credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |-----------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | -| Access Key | | | Specifies the AWS Access Key | -| Secret Key | | | Specifies the AWS Secret Key | +| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| Access Key | | | Specifies the AWS Access Key. | +| Secret Key | | | Specifies the AWS Secret Key. | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -48,13 +55,11 @@ properties (not in bold) are considered optional. ### Description -Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single -controller service so that Azure storage credentials can be managed and controlled in a central location. +Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single controller service so that Azure storage credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |----------------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -63,59 +68,251 @@ properties (not in bold) are considered optional. | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used. | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions). | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used. | -| **Use Managed Identity Credentials** | false | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties. | +| **Use Managed Identity Credentials** | false | true
false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | -## GCPCredentialsControllerService + +## ElasticsearchCredentialsControllerService ### Description -Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors to reference this single -controller service so that Google Cloud Platform credentials can be managed and controlled in a central location. +Elasticsearch/Opensearch Credentials Controller Service ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. +| Name | Default Value | Allowable Values | Description | +|----------|---------------|------------------|----------------------------------------------------------------------------------| +| Username | | | The username for basic authentication
**Supports Expression Language: true** | +| Password | | | The password for basic authentication
**Supports Expression Language: true** | +| API Key | | | The API Key to use | -| Name | Default Value | Allowable Values | Description | -|---------------------------|----------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| -| **Credentials Location** | Google Application Default Credentials | Google Application Default Credentials
Use Compute Engine Credentials
Service Account JSON File
Service Account JSON
Use Anonymous credentials | The location of the credentials. | -| Service Account JSON File | | | Path to a file containing a Service Account key file in JSON format. | -| Service Account JSON | | | The raw JSON containing a Service Account keyfile. | +## ExecuteJavaControllerService -## ElasticsearchCredentialsControllerService +### Description + +ExecuteJavaClass runs NiFi Controller services given a provided system path + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------------|---------------|------------------|-------------------------------------------------| +| **NiFi Controller Service** | | | Name of NiFi Controller Service to load and run | + + +## GCPCredentialsControllerService ### Description -Elasticsearch Credentials Controller Service +Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors to reference this single controller service so that Google Cloud Platform credentials can be managed and controlled in a central location. ### Properties -In the list below, the names of required properties appear in bold. Any other -properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. +| Name | Default Value | Allowable Values | Description | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| +| **Credentials Location** | Google Application Default Credentials | Google Application Default Credentials
Use Compute Engine Credentials
Service Account JSON File
Service Account JSON
Use Anonymous credentials | The location of the credentials. | +| Service Account JSON File | | | Path to a file containing a Service Account key file in JSON format. | +| Service Account JSON | | | The raw JSON containing a Service Account keyfile. | -| Name | Default Value | Allowable Values | Description | -|------------------------|---------------|----------------------------------|---------------------------------------| -| Username | | | The username for basic authentication | -| Password | | | The password for basic authentication | -| API Key | | | The API Key to use | + +## JavaControllerService + +### Description + +Allows specification of nars to be used within referenced processors. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------|---------------|------------------|-----------------------------------------------| +| **Nar Directory** | | | Directory containing the nars to deploy | +| **Nar Deployment Directory** | | | Directory in which nars will be deployed | +| **Nar Document Directory** | | | Directory in which documents will be deployed | ## KubernetesControllerService ### Description -Controller service that provides access to the Kubernetes API. +Controller service that provides access to the Kubernetes API ### Properties -In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. | Name | Default Value | Allowable Values | Description | |-----------------------|---------------|------------------|----------------------------------------------------------------------------------------------| | Namespace Filter | default | | Limit the output to pods in namespaces which match this regular expression | | Pod Name Filter | | | If present, limit the output to pods the name of which matches this regular expression | | Container Name Filter | | | If present, limit the output to containers the name of which matches this regular expression | + + +## LinuxPowerManagerService + +### Description + +Linux power management service that enables control of power usage in the agent through Linux power management information. Use name "ThreadPoolManager" to throttle battery consumption + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------|---------------------------------------|------------------|--------------------------------------------------------------------------------------------| +| **Battery Capacity Path** | /sys/class/power_supply/BAT0/capacity | | Path to the battery level | +| **Battery Status Path** | /sys/class/power_supply/BAT0/status | | Path to the battery status ( Discharging/Battery ) | +| **Battery Status Discharge** | Discharging | | Keyword to identify if battery is discharging | +| **Trigger Threshold** | 75 | | Battery threshold before which we consider a slow reduction. Should be a number from 1-100 | +| **Low Battery Threshold** | 50 | | Battery threshold before which we will aggressively reduce. Should be a number from 1-100 | +| **Wait Period** | 100 ms | | Decay between checking threshold and determining if a reduction is needed | + + +## NetworkPrioritizerService + +### Description + +Enables selection of networking interfaces on defined parameters to include output and payload size + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|---------------------------------------------------------------------------------------| +| Network Controllers | | | Comma separated list of network controllers in order of priority for this prioritizer | +| **Max Throughput** | 1 MB | | Max throughput ( per second ) for these network controllers | +| **Max Payload** | 1 GB | | Maximum payload for these network controllers | +| **Verify Interfaces** | true | true
false | Verify that interfaces are operational | +| Default Prioritizer | false | true
false | Sets this controller service as the default prioritizer for all comms | + + +## ODBCService + +### Description + +Controller service that provides ODBC database connection + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|----------------------------| +| **Connection String** | | | Database Connection String | + + +## PersistentMapStateStorage + +### Description + +A persistable state storage service implemented by a locked std::unordered_map and persisted into a file + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| Always Persist | false | true
false | Persist every change instead of persisting it periodically. | +| Auto Persistence Interval | 1 min | | The interval of the periodic task persisting all values. Only used if Always Persist is false. If set to 0 seconds, auto persistence will be disabled. | +| **File** | | | Path to a file to store state | + + +## RocksDbStateStorage + +### Description + +A state storage service implemented by RocksDB + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| Always Persist | false | true
false | Persist every change instead of persisting it periodically. | +| Auto Persistence Interval | 1 min | | The interval of the periodic task persisting all values. Only used if Always Persist is false. If set to 0 seconds, auto persistence will be disabled. | +| **Directory** | | | Path to a directory for the database | + + +## SmbConnectionControllerService + +### Description + +SMB Connection Controller Service + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------| +| **Hostname** | | | The network host to which files should be written. | +| **Share** | | | The network share to which files should be written. This is the "first folder" after the hostname: \\hostname\[share]\dir1\dir2 | +| Domain | | | The domain used for authentication. Optional, in most cases username and password is sufficient. | +| Username | | | The username used for authentication. If no username is set then anonymous authentication is attempted. | +| Password | | | The password used for authentication. Required if Username is set. | + + +## SSLContextService + +### Description + +Controller service that provides SSL/TLS capabilities to consuming interfaces + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| Certificate Store Location | LocalMachine | CurrentUser
LocalMachine
CurrentService
Services
Users
CurrentUserGroupPolicy
LocalMachineGroupPolicy
LocalMachineEnterprise | One of the Windows certificate store locations, eg. LocalMachine or CurrentUser (Windows only) | +| Server Cert Store | ROOT | | The name of the certificate store which contains the server certificate (Windows only) | +| Client Cert Store | MY | | The name of the certificate store which contains the client certificate (Windows only) | +| Client Cert CN | | | The CN that the client certificate is required to match; default: use the first available client certificate in the store (Windows only) | +| Client Cert Key Usage | Client Authentication | | Comma-separated list of enhanced key usage values that the client certificate is required to have (Windows only) | +| Client Certificate | | | Client Certificate | +| Private Key | | | Private Key file | +| Passphrase | | | Client passphrase. Either a file or unencrypted text | +| CA Certificate | | | CA certificate file | +| Use System Cert Store | false | true
false | Whether to use the certificates in the OS's certificate store | + + +## UpdatePolicyControllerService + +### Description + +UpdatePolicyControllerService allows a flow specific policy on allowing or disallowing updates. Since the flow dictates the purpose of a device it will also be used to dictate updates to specific components. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|-----------------------------------------------------------------------| +| Allow All Properties | false | true
false | Allows all properties, which are also not disallowed, to be updated | +| Persist Updates | false | true
false | Property that dictates whether updates should persist after a restart | +| Allowed Properties | | | Properties for which we will allow updates | +| Disallowed Properties | | | Properties for which we will not allow updates | + + +## VolatileMapStateStorage + +### Description + +A key-value service implemented by a locked std::unordered_map + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------|---------------|------------------|--------------------------------| +| Linked Services | | | Referenced Controller Services | diff --git a/PROCESSORS.md b/PROCESSORS.md index 6da9203df7..940f0163f9 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -15,91 +15,91 @@ limitations under the License. ## Table of Contents -- [AppendHostInfo](#appendhostinfo) -- [ApplyTemplate](#applytemplate) -- [AttributesToJSON](#attributestojson) -- [BinFiles](#binfiles) -- [CapturePacket](#capturepacket) -- [CaptureRTSPFrame](#capturertspframe) -- [CollectKubernetesPodMetrics](#collectkubernetespodmetrics) -- [CollectorInitiatedSubscription](#collectorinitiatedsubscription) -- [CompressContent](#compresscontent) -- [ConsumeJournald](#consumejournald) -- [ConsumeKafka](#consumekafka) -- [ConsumeMQTT](#consumemqtt) -- [ConsumeWindowsEventLog](#consumewindowseventlog) -- [DefragmentText](#defragmenttext) -- [DeleteAzureBlobStorage](#deleteazureblobstorage) -- [DeleteAzureDataLakeStorage](#deleteazuredatalakestorage) -- [DeleteGCSObject](#deletegcsobject) -- [DeleteS3Object](#deletes3object) -- [ExecuteJavaProcessor](#executejavaprocessor) -- [ExecuteProcess](#executeprocess) -- [ExecutePythonProcessor](#executepythonprocessor) -- [ExecuteScript](#executescript) -- [ExecuteSQL](#executesql) -- [ExtractText](#extracttext) -- [FetchAzureBlobStorage](#fetchazureblobstorage) -- [FetchAzureDataLakeStorage](#fetchazuredatalakestorage) -- [FetchFile](#fetchfile) -- [FetchGCSObject](#fetchgcsobject) -- [FetchOPCProcessor](#fetchopcprocessor) -- [FetchS3Object](#fetchs3object) -- [FetchSFTP](#fetchsftp) -- [FetchSmb](#fetchsmb) -- [FocusArchiveEntry](#focusarchiveentry) -- [GenerateFlowFile](#generateflowfile) -- [GetEnvironmentalSensors](#getenvironmentalsensors) -- [GetFile](#getfile) -- [GetGPS](#getgps) -- [GetMovementSensors](#getmovementsensors) -- [GetTCP](#gettcp) -- [GetUSBCamera](#getusbcamera) -- [HashContent](#hashcontent) -- [InvokeHTTP](#invokehttp) -- [ListAzureBlobStorage](#listazureblobstorage) -- [ListAzureDataLakeStorage](#listazuredatalakestorage) -- [ListenHTTP](#listenhttp) -- [ListenSyslog](#listensyslog) -- [ListenTCP](#listentcp) -- [ListenUDP](#listenudp) -- [ListFile](#listfile) -- [ListGCSBucket](#listgcsbucket) -- [ListS3](#lists3) -- [ListSFTP](#listsftp) -- [ListSmb](#listsmb) -- [LogAttribute](#logattribute) -- [ManipulateArchive](#manipulatearchive) -- [MergeContent](#mergecontent) -- [MotionDetector](#motiondetector) -- [PerformanceDataMonitor](#performancedatamonitor) -- [PostElasticsearch](#postelasticsearch) -- [ProcFsMonitor](#procfsmonitor) -- [PublishKafka](#publishkafka) -- [PublishMQTT](#publishmqtt) -- [PutAzureBlobStorage](#putazureblobstorage) -- [PutAzureDataLakeStorage](#putazuredatalakestorage) -- [PutFile](#putfile) -- [PutGCSObject](#putgcsobject) -- [PutOPCProcessor](#putopcprocessor) -- [PutS3Object](#puts3object) -- [PutSFTP](#putsftp) -- [PutSmb](#putsmb) -- [PutSplunkHTTP](#putsplunkhttp) -- [PutSQL](#putsql) -- [PutTCP](#puttcp) -- [PutUDP](#putudp) -- [QueryDatabaseTable](#querydatabasetable) -- [QuerySplunkIndexingStatus](#querysplunkindexingstatus) -- [ReplaceText](#replacetext) -- [RetryFlowFile](#retryflowfile) -- [RouteOnAttribute](#routeonattribute) -- [RouteText](#routetext) -- [SourceInitiatedSubscriptionListener](#sourceinitiatedsubscriptionlistener) -- [TailEventLog](#taileventlog) -- [TailFile](#tailfile) -- [UnfocusArchiveEntry](#unfocusarchiveentry) -- [UpdateAttribute](#updateattribute) +- [AppendHostInfo](#AppendHostInfo) +- [ApplyTemplate](#ApplyTemplate) +- [AttributesToJSON](#AttributesToJSON) +- [BinFiles](#BinFiles) +- [CapturePacket](#CapturePacket) +- [CaptureRTSPFrame](#CaptureRTSPFrame) +- [CollectKubernetesPodMetrics](#CollectKubernetesPodMetrics) +- [CollectorInitiatedSubscription](#CollectorInitiatedSubscription) +- [CompressContent](#CompressContent) +- [ConsumeJournald](#ConsumeJournald) +- [ConsumeKafka](#ConsumeKafka) +- [ConsumeMQTT](#ConsumeMQTT) +- [ConsumeWindowsEventLog](#ConsumeWindowsEventLog) +- [DefragmentText](#DefragmentText) +- [DeleteAzureBlobStorage](#DeleteAzureBlobStorage) +- [DeleteAzureDataLakeStorage](#DeleteAzureDataLakeStorage) +- [DeleteGCSObject](#DeleteGCSObject) +- [DeleteS3Object](#DeleteS3Object) +- [ExecuteJavaProcessor](#ExecuteJavaProcessor) +- [ExecuteProcess](#ExecuteProcess) +- [ExecutePythonProcessor](#ExecutePythonProcessor) +- [ExecuteScript](#ExecuteScript) +- [ExecuteSQL](#ExecuteSQL) +- [ExtractText](#ExtractText) +- [FetchAzureBlobStorage](#FetchAzureBlobStorage) +- [FetchAzureDataLakeStorage](#FetchAzureDataLakeStorage) +- [FetchFile](#FetchFile) +- [FetchGCSObject](#FetchGCSObject) +- [FetchOPCProcessor](#FetchOPCProcessor) +- [FetchS3Object](#FetchS3Object) +- [FetchSFTP](#FetchSFTP) +- [FetchSmb](#FetchSmb) +- [FocusArchiveEntry](#FocusArchiveEntry) +- [GenerateFlowFile](#GenerateFlowFile) +- [GetEnvironmentalSensors](#GetEnvironmentalSensors) +- [GetFile](#GetFile) +- [GetGPS](#GetGPS) +- [GetMovementSensors](#GetMovementSensors) +- [GetTCP](#GetTCP) +- [GetUSBCamera](#GetUSBCamera) +- [HashContent](#HashContent) +- [InvokeHTTP](#InvokeHTTP) +- [ListAzureBlobStorage](#ListAzureBlobStorage) +- [ListAzureDataLakeStorage](#ListAzureDataLakeStorage) +- [ListenHTTP](#ListenHTTP) +- [ListenSyslog](#ListenSyslog) +- [ListenTCP](#ListenTCP) +- [ListenUDP](#ListenUDP) +- [ListFile](#ListFile) +- [ListGCSBucket](#ListGCSBucket) +- [ListS3](#ListS3) +- [ListSFTP](#ListSFTP) +- [ListSmb](#ListSmb) +- [LogAttribute](#LogAttribute) +- [ManipulateArchive](#ManipulateArchive) +- [MergeContent](#MergeContent) +- [MotionDetector](#MotionDetector) +- [PerformanceDataMonitor](#PerformanceDataMonitor) +- [PostElasticsearch](#PostElasticsearch) +- [ProcFsMonitor](#ProcFsMonitor) +- [PublishKafka](#PublishKafka) +- [PublishMQTT](#PublishMQTT) +- [PutAzureBlobStorage](#PutAzureBlobStorage) +- [PutAzureDataLakeStorage](#PutAzureDataLakeStorage) +- [PutFile](#PutFile) +- [PutGCSObject](#PutGCSObject) +- [PutOPCProcessor](#PutOPCProcessor) +- [PutS3Object](#PutS3Object) +- [PutSFTP](#PutSFTP) +- [PutSmb](#PutSmb) +- [PutSplunkHTTP](#PutSplunkHTTP) +- [PutSQL](#PutSQL) +- [PutTCP](#PutTCP) +- [PutUDP](#PutUDP) +- [QueryDatabaseTable](#QueryDatabaseTable) +- [QuerySplunkIndexingStatus](#QuerySplunkIndexingStatus) +- [ReplaceText](#ReplaceText) +- [RetryFlowFile](#RetryFlowFile) +- [RouteOnAttribute](#RouteOnAttribute) +- [RouteText](#RouteText) +- [SourceInitiatedSubscriptionListener](#SourceInitiatedSubscriptionListener) +- [TailEventLog](#TailEventLog) +- [TailFile](#TailFile) +- [UnfocusArchiveEntry](#UnfocusArchiveEntry) +- [UpdateAttribute](#UpdateAttribute) ## AppendHostInfo @@ -860,14 +860,14 @@ Reads the contents of a file from disk and streams it into the contents of an in In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|--------------------------------------|---------------|-----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| File to Fetch | | | The fully-qualified filename of the file to fetch from the file system. If not defined the default ${absolute.path}/${filename} path is used.
**Supports Expression Language: true** | -| **Completion Strategy** | None | None
Move File
Delete File | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | -| Move Destination Directory | | | The directory to move the original file to once it has been fetched from the file system. This property is ignored unless the Completion Strategy is set to "Move File". If the directory does not exist, it will be created.
**Supports Expression Language: true** | -| **Move Conflict Strategy** | Rename | Rename
Replace File
Keep Existing
Fail | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | -| **Log level when file not found** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
OFF | Log level to use in case the file does not exist when the processor is triggered | -| **Log level when permission denied** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
OFF | Log level to use in case agent does not have sufficient permissions to read the file | +| Name | Default Value | Allowable Values | Description | +|--------------------------------------|---------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| File to Fetch | | | The fully-qualified filename of the file to fetch from the file system. If not defined the default ${absolute.path}/${filename} path is used.
**Supports Expression Language: true** | +| **Completion Strategy** | None | None
Move File
Delete File | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | +| Move Destination Directory | | | The directory to move the original file to once it has been fetched from the file system. This property is ignored unless the Completion Strategy is set to "Move File". If the directory does not exist, it will be created.
**Supports Expression Language: true** | +| **Move Conflict Strategy** | Rename | Rename
Replace File
Keep Existing
Fail | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | +| **Log level when file not found** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
CRITICAL
OFF | Log level to use in case the file does not exist when the processor is triggered | +| **Log level when permission denied** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
CRITICAL
OFF | Log level to use in case agent does not have sufficient permissions to read the file | ### Relationships @@ -1042,7 +1042,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use FetchFile instead. | | Input Directory | | | The full path of the file to be retrieved from the remote server. If left empty, the path and filename attributes of the incoming flow file will be used.
**Supports Expression Language: true** | ### Relationships @@ -1732,18 +1732,18 @@ Retrieves a listing of files from an SMB share. For each file that is listed, cr In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | -| Input Directory | | | The input directory to list the contents of | -| **Recurse Subdirectories** | true | | Indicates whether to list files from subdirectories of the directory | -| File Filter | | | Only files whose names match the given regular expression will be picked up | -| Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | -| **Minimum File Age** | 5 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | -| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | -| Minimum File Size | | | The minimum size that a file must be in order to be pulled | -| Maximum File Size | | | The maximum size that a file can be in order to be pulled | -| **Ignore Hidden Files** | true | | Indicates whether or not hidden files should be ignored | +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use ListFile instead. | +| Input Directory | | | The input directory to list the contents of | +| **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory | +| File Filter | | | Only files whose names match the given regular expression will be picked up | +| Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | +| **Minimum File Age** | 5 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file must be in order to be pulled | +| Maximum File Size | | | The maximum size that a file can be in order to be pulled | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | ### Relationships @@ -1753,15 +1753,15 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| filename | success | The name of the file that was read from filesystem. | -| path | success | The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file then the path attribute will be set to "sub/folder". | -| serviceLocation | success | The SMB URL of the share. | -| lastModifiedTime | success | The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| creationTime | success | The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| lastAccessTime | success | The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | -| size | success | The size of the file in bytes. | +| Attribute | Relationship | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| filename | success | The name of the file that was read from filesystem. | +| path | success | The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file then the path attribute will be set to "DIRECTORY/sub/folder". | +| serviceLocation | success | The SMB URL of the share. | +| lastModifiedTime | success | The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| creationTime | success | The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| lastAccessTime | success | The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| size | success | The size of the file in bytes. | ## LogAttribute @@ -2227,7 +2227,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Directory | . | | The output directory to which to put files
**Supports Expression Language: true** | -| Conflict Resolution Strategy | fail | fail
ignore
replace | Indicates what should happen when a file with the same name already exists in the output directory | +| Conflict Resolution Strategy | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | | **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | | Maximum File Count | -1 | | Specifies the maximum number of files that can exist in the output directory | @@ -2445,12 +2445,12 @@ Writes the contents of a FlowFile to an smb network location In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------| -| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | -| Directory | . | | The output directory to which to put files
**Supports Expression Language: true** | -| Conflict Resolution Strategy | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | -| **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. If the SMB share is auto-mounted to a drive letter, its recommended to use PutFile instead. | +| Directory | . | | The output directory to which to put files
**Supports Expression Language: true** | +| Conflict Resolution Strategy | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | +| **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | ### Relationships diff --git a/extensions/aws/controllerservices/AWSCredentialsService.h b/extensions/aws/controllerservices/AWSCredentialsService.h index 46049572f7..5b2beaf224 100644 --- a/extensions/aws/controllerservices/AWSCredentialsService.h +++ b/extensions/aws/controllerservices/AWSCredentialsService.h @@ -47,7 +47,9 @@ class AWSCredentialsService : public core::controller::ControllerService { : ControllerService(name) { } - EXTENSIONAPI static constexpr const char* Description = "AWS Credentials Management Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the Amazon Web Services (AWS) credentials for an AWS account. This allows for multiple " + "AWS credential services to be defined. This also allows for multiple AWS related processors to reference this single " + "controller service so that AWS credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials") .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.") diff --git a/extensions/azure/controllerservices/AzureStorageCredentialsService.h b/extensions/azure/controllerservices/AzureStorageCredentialsService.h index 968be0aa9c..4ab86fab19 100644 --- a/extensions/azure/controllerservices/AzureStorageCredentialsService.h +++ b/extensions/azure/controllerservices/AzureStorageCredentialsService.h @@ -34,7 +34,8 @@ namespace org::apache::nifi::minifi::azure::controllers { class AzureStorageCredentialsService : public core::controller::ControllerService { public: - EXTENSIONAPI static constexpr const char* Description = "Azure Storage Credentials Management Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the credentials for an Azure Storage account. This allows for multiple Azure Storage related processors to reference this single " + "controller service so that Azure storage credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto StorageAccountName = core::PropertyDefinitionBuilder<>::createProperty("Storage Account Name") .withDescription("The storage account name.") diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h index a4f217b313..8388415258 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h @@ -64,7 +64,8 @@ namespace org::apache::nifi::minifi::extensions::gcp { class GCPCredentialsControllerService : public core::controller::ControllerService { public: - EXTENSIONAPI static constexpr const char* Description = "Google Cloud Platform Credentials Controller Service"; + EXTENSIONAPI static constexpr const char* Description = "Manages the credentials for Google Cloud Platform. This allows for multiple Google Cloud Platform related processors " + "to reference this single controller service so that Google Cloud Platform credentials can be managed and controlled in a central location."; EXTENSIONAPI static constexpr auto CredentialsLoc = core::PropertyDefinitionBuilder()>::createProperty("Credentials Location") .withDescription("The location of the credentials.") diff --git a/libminifi/include/controllers/SSLContextService.h b/libminifi/include/controllers/SSLContextService.h index 1c2c74a412..c3523cd8d6 100644 --- a/libminifi/include/controllers/SSLContextService.h +++ b/libminifi/include/controllers/SSLContextService.h @@ -177,27 +177,27 @@ class SSLContextService : public core::controller::ControllerService { #ifdef WIN32 MINIFIAPI static constexpr auto CertStoreLocation = core::PropertyDefinitionBuilder::createProperty("Certificate Store Location") - .withDescription("One of the Windows certificate store locations, eg. LocalMachine or CurrentUser") + .withDescription("One of the Windows certificate store locations, eg. LocalMachine or CurrentUser (Windows only)") .withAllowedValues(utils::tls::WindowsCertStoreLocation::LOCATION_NAMES) .isRequired(false) .withDefaultValue(utils::tls::WindowsCertStoreLocation::DEFAULT_LOCATION) .build(); MINIFIAPI static constexpr auto ServerCertStore = core::PropertyDefinitionBuilder<>::createProperty("Server Cert Store") - .withDescription("The name of the certificate store which contains the server certificate") + .withDescription("The name of the certificate store which contains the server certificate (Windows only)") .isRequired(false) .withDefaultValue("ROOT") .build(); MINIFIAPI static constexpr auto ClientCertStore = core::PropertyDefinitionBuilder<>::createProperty("Client Cert Store") - .withDescription("The name of the certificate store which contains the client certificate") + .withDescription("The name of the certificate store which contains the client certificate (Windows only)") .isRequired(false) .withDefaultValue("MY") .build(); MINIFIAPI static constexpr auto ClientCertCN = core::PropertyDefinitionBuilder<>::createProperty("Client Cert CN") - .withDescription("The CN that the client certificate is required to match; default: use the first available client certificate in the store") + .withDescription("The CN that the client certificate is required to match; default: use the first available client certificate in the store (Windows only)") .isRequired(false) .build(); MINIFIAPI static constexpr auto ClientCertKeyUsage = core::PropertyDefinitionBuilder<>::createProperty("Client Cert Key Usage") - .withDescription("Comma-separated list of enhanced key usage values that the client certificate is required to have") + .withDescription("Comma-separated list of enhanced key usage values that the client certificate is required to have (Windows only)") .isRequired(false) .withDefaultValue("Client Authentication") .build(); diff --git a/libminifi/include/core/PropertyDefinition.h b/libminifi/include/core/PropertyDefinition.h index 1af98de75d..e1a6d8a12c 100644 --- a/libminifi/include/core/PropertyDefinition.h +++ b/libminifi/include/core/PropertyDefinition.h @@ -39,7 +39,7 @@ struct PropertyDefinition { std::array dependent_properties; std::array, NumExclusiveOfProperties> exclusive_of_properties; std::optional default_value; - gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); bool supports_expression_language = false; }; @@ -53,7 +53,7 @@ struct PropertyReference { std::span dependent_properties; std::span> exclusive_of_properties; std::optional default_value; - gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); bool supports_expression_language = false; constexpr PropertyReference() = default; diff --git a/libminifi/include/core/PropertyType.h b/libminifi/include/core/PropertyType.h index 1a75132e99..880e35fdf8 100644 --- a/libminifi/include/core/PropertyType.h +++ b/libminifi/include/core/PropertyType.h @@ -335,21 +335,21 @@ inline constexpr auto LISTEN_PORT_TYPE = ListenPortValidator{}; inline gsl::not_null getValidator(const std::shared_ptr& input) { if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&DATA_SIZE_TYPE); + return gsl::make_not_null(&DATA_SIZE_TYPE); } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&TIME_PERIOD_TYPE); + return gsl::make_not_null(&TIME_PERIOD_TYPE); } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&BOOLEAN_TYPE); + return gsl::make_not_null(&BOOLEAN_TYPE); } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&INTEGER_TYPE); + return gsl::make_not_null(&INTEGER_TYPE); } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&UNSIGNED_INT_TYPE);; + return gsl::make_not_null(&UNSIGNED_INT_TYPE);; } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&LONG_TYPE); + return gsl::make_not_null(&LONG_TYPE); } else if (std::dynamic_pointer_cast(input) != nullptr) { - return gsl::make_not_null(&UNSIGNED_LONG_TYPE); + return gsl::make_not_null(&UNSIGNED_LONG_TYPE); } else { - return gsl::make_not_null(&VALID_TYPE); + return gsl::make_not_null(&VALID_TYPE); } } } // namespace StandardPropertyTypes diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index bd1c6a8d9b..91cfead48c 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -17,16 +17,17 @@ #include "AgentDocs.h" -#include #include #include #include #include +#include "range/v3/algorithm.hpp" #include "range/v3/action/transform.hpp" +#include "range/v3/algorithm/lexicographical_compare.hpp" +#include "range/v3/range/conversion.hpp" #include "range/v3/view/transform.hpp" #include "range/v3/view/join.hpp" -#include "range/v3/range/conversion.hpp" #include "agent/agent_docs.h" #include "agent/agent_version.h" @@ -51,7 +52,7 @@ std::string formatName(std::string_view name_view, bool is_required) { } std::string formatAllowedValues(const minifi::core::Property& property) { - if (&property.getValidator() == &minifi::core::StandardPropertyTypes::BOOLEAN_TYPE) { + if (property.getValidator().getValidatorName() == minifi::core::StandardPropertyTypes::BOOLEAN_TYPE.getValidatorName()) { return "true
false"; } else { const auto allowed_values = property.getAllowedValues(); @@ -64,7 +65,7 @@ std::string formatAllowedValues(const minifi::core::Property& property) { std::string formatDescription(std::string_view description_view, bool supports_expression_language = false) { std::string description{description_view}; - org::apache::nifi::minifi::utils::StringUtils::replaceAll(description, "\n", "
"); + minifi::utils::StringUtils::replaceAll(description, "\n", "
"); return supports_expression_language ? description + "
**Supports Expression Language: true**" : description; } @@ -72,122 +73,137 @@ std::string formatListOfRelationships(std::span"; + +void writeHeader(std::ostream& docs, const std::vector>& class_descriptions) { + docs << APACHE_LICENSE; + + docs << "\n\n## Table of Contents\n\n"; + for (const auto& [name, documentation] : class_descriptions) { + docs << "- [" << name << "](#" << name << ")\n"; + } +} -namespace org::apache::nifi::minifi::docs { +void writeName(std::ostream& docs, std::string_view name) { + docs << "\n\n## " << name; +} + +void writeDescription(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n\n### Description\n\n"; + docs << documentation.description_; +} -std::string AgentDocs::extractClassName(const std::string &processor) { - auto positionOfLastDot = processor.find_last_of('.'); - if (positionOfLastDot != std::string::npos) { - return processor.substr(positionOfLastDot + 1); +void writeProperties(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n\n### Properties"; + docs << "\n\nIn the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. " + << "The table also indicates any default values, and whether a property supports the NiFi Expression Language."; + minifi::docs::Table properties{{"Name", "Default Value", "Allowable Values", "Description"}}; + for (const auto &property : documentation.class_properties_) { + properties.addRow({ + formatName(property.getName(), property.getRequired()), + property.getDefaultValue().to_string(), + formatAllowedValues(property), + formatDescription(property.getDescription(), property.supportsExpressionLanguage()) + }); } - return processor; + docs << "\n\n" << properties.toString(); } -void AgentDocs::generate(const std::filesystem::path& docsdir, std::ostream &genStream) { - std::map processorSet; - for (const auto &group : minifi::AgentBuild::getExtensions()) { - struct Components descriptions = build_description_.getClassDescriptions(group); - for (const auto& processor_description : descriptions.processors_) { - processorSet.insert(std::make_pair(extractClassName(processor_description.full_name_), processor_description)); - } +void writeDynamicProperties(std::ostream& docs, const minifi::ClassDescription& documentation) { + if (documentation.dynamic_properties_.empty()) { return; } + + docs << "\n### Dynamic Properties\n\n"; + minifi::docs::Table dynamic_properties{{"Name", "Value", "Description"}}; + for (const auto &dynamic_property : documentation.dynamic_properties_) { + dynamic_properties.addRow({ + formatName(dynamic_property.name, false), + std::string(dynamic_property.value), + formatDescription(dynamic_property.description, dynamic_property.supports_expression_language) + }); } - for (const auto &processor : processorSet) { - const auto& filename = docsdir / processor.first; - std::ofstream outfile(filename); - - outfile << "## " << processor.first << "\n\n"; - outfile << "### Description\n\n"; - outfile << processor.second.description_ << '\n'; - - outfile << "\n### Properties\n\n"; - outfile << "In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. " - << "The table also indicates any default values, and whether a property supports the NiFi Expression Language.\n\n"; - - Table properties{{"Name", "Default Value", "Allowable Values", "Description"}}; - for (const auto &prop : processor.second.class_properties_) { - properties.addRow({ - formatName(prop.getName(), prop.getRequired()), - prop.getDefaultValue().to_string(), - formatAllowedValues(prop), - formatDescription(prop.getDescription(), prop.supportsExpressionLanguage())}); - } - outfile << properties.toString() << '\n'; - - if (!processor.second.dynamic_properties_.empty()) { - outfile << "### Dynamic Properties\n\n"; - Table dynamic_properties{{"Name", "Value", "Description"}}; - for (const auto& dynamic_property : processor.second.dynamic_properties_) { - dynamic_properties.addRow({ - formatName(dynamic_property.name, false), - std::string(dynamic_property.value), - formatDescription(dynamic_property.description, dynamic_property.supports_expression_language) - }); - } - outfile << dynamic_properties.toString() << '\n'; - } + docs << dynamic_properties.toString(); +} - outfile << "### Relationships\n\n"; - Table relationships{{"Name", "Description"}}; - for (const auto &rel : processor.second.class_relationships_) { - relationships.addRow({rel.getName(), formatDescription(rel.getDescription())}); - } - outfile << relationships.toString() << '\n'; - - if (!processor.second.output_attributes_.empty()) { - outfile << "### Output Attributes\n\n"; - Table output_attributes{{"Attribute", "Relationship", "Description"}}; - for (const auto& output_attribute : processor.second.output_attributes_) { - output_attributes.addRow({ - std::string(output_attribute.name), - formatListOfRelationships(output_attribute.relationships), - formatDescription(output_attribute.description)}); - } - outfile << output_attributes.toString() << '\n'; - } +void writeRelationships(std::ostream& docs, const minifi::ClassDescription& documentation) { + docs << "\n### Relationships\n\n"; + minifi::docs::Table relationships{{"Name", "Description"}}; + for (const auto &rel : documentation.class_relationships_) { + relationships.addRow({rel.getName(), formatDescription(rel.getDescription())}); } + docs << relationships.toString(); +} + +void writeOutputAttributes(std::ostream& docs, const minifi::ClassDescription& documentation) { + if (documentation.output_attributes_.empty()) { return; } + + docs << "\n### Output Attributes"; + minifi::docs::Table output_attributes{{"Attribute", "Relationship", "Description"}}; + for (const auto &output_attribute : documentation.output_attributes_) { + output_attributes.addRow({ + std::string(output_attribute.name), + formatListOfRelationships(output_attribute.relationships), + formatDescription(output_attribute.description)}); + } + docs << "\n\n" << output_attributes.toString(); +} - std::map fileList; - auto fileFind = [&fileList](const std::filesystem::path& base_path, const std::filesystem::path& file) -> bool { - if (file.string().find(".extra") == std::string::npos) { - auto file_name = file.string(); - ranges::actions::transform(file_name, [](auto ch) { return ::tolower(static_cast(ch)); }); - fileList.emplace(file_name, base_path / file); +std::string extractClassName(const std::string& full_class_name) { + return minifi::utils::StringUtils::split(full_class_name, ".").back(); +} + +constexpr auto LowercaseFirst = [](const auto& key_value) { + return minifi::utils::StringUtils::toLower(key_value.first); +}; + +} // namespace + +namespace org::apache::nifi::minifi::docs { + +void AgentDocs::generate(const std::filesystem::path& docs_dir) { + std::vector> controller_services; + std::vector> processors; + for (const auto &group : minifi::AgentBuild::getExtensions()) { + struct Components descriptions = build_description_.getClassDescriptions(group); + for (const auto &controller_service_description : descriptions.controller_services_) { + controller_services.emplace_back(extractClassName(controller_service_description.full_name_), controller_service_description); + } + for (const auto &processor_description : descriptions.processors_) { + processors.emplace_back(extractClassName(processor_description.full_name_), processor_description); } - return true; - }; - utils::file::list_dir(docsdir, fileFind, core::logging::LoggerFactory::getLogger()); - - genStream << "\n\n"; - genStream << "## Table of Contents\n\n"; - - for (const auto& file : fileList) { - genStream << "- [" << file.second.filename().string() << "](#" << file.first << ")\n"; } - genStream << "\n\n"; - - for (const auto& file : fileList) { - std::ifstream filestream(file.second); - genStream << filestream.rdbuf() << '\n'; - auto extra_path = file.second; - extra_path += ".extra"; - std::ifstream filestreamExtra(extra_path); - if (filestreamExtra.good()) { - genStream << filestreamExtra.rdbuf() << '\n'; - } + ranges::sort(controller_services, std::less(), LowercaseFirst); + ranges::sort(processors, std::less(), LowercaseFirst); + + std::ofstream controllers_md(docs_dir / "CONTROLLERS.md"); + writeHeader(controllers_md, controller_services); + for (const auto& [name, documentation] : controller_services) { + writeName(controllers_md, name); + writeDescription(controllers_md, documentation); + writeProperties(controllers_md, documentation); + } + + std::ofstream processors_md(docs_dir / "PROCESSORS.md"); + writeHeader(processors_md, processors); + for (const auto& [name, documentation] : processors) { + writeName(processors_md, name); + writeDescription(processors_md, documentation); + writeProperties(processors_md, documentation); + writeDynamicProperties(processors_md, documentation); + writeRelationships(processors_md, documentation); + writeOutputAttributes(processors_md, documentation); } } diff --git a/minifi_main/AgentDocs.h b/minifi_main/AgentDocs.h index e2017a87cc..276aef763c 100644 --- a/minifi_main/AgentDocs.h +++ b/minifi_main/AgentDocs.h @@ -14,8 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MAIN_AGENTDOCS_H_ -#define MAIN_AGENTDOCS_H_ +#pragma once #include #include "agent/build_description.h" @@ -24,12 +23,10 @@ namespace org::apache::nifi::minifi::docs { class AgentDocs { public: - void generate(const std::filesystem::path &docsdir, std::ostream &genStream); + void generate(const std::filesystem::path& docs_dir); + private: - [[nodiscard]] static std::string extractClassName(const std::string &processor); BuildDescription build_description_; }; } // namespace org::apache::nifi::minifi::docs - -#endif // MAIN_AGENTDOCS_H_ diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index 6e34b93604..9d1d9b02f8 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -116,7 +116,7 @@ void sigHandler(int signal) { } } -void dumpDocs(const std::shared_ptr &configuration, const std::string &dir, std::ostream &out) { +void dumpDocs(const std::shared_ptr &configuration, const std::string &dir) { auto pythoncreator = core::ClassLoader::getDefaultClassLoader().instantiate("PythonCreator", "PythonCreator"); if (nullptr != pythoncreator) { pythoncreator->configure(configuration); @@ -124,7 +124,7 @@ void dumpDocs(const std::shared_ptr &configuration, const std minifi::docs::AgentDocs docsCreator; - docsCreator.generate(dir, out); + docsCreator.generate(dir); } void writeJsonSchema(const std::shared_ptr &configuration, std::ostream& out) { @@ -160,22 +160,7 @@ void dumpDocsIfRequested(const argparse::ArgumentParser& parser, const std::shar } std::cout << "Dumping docs to " << docs_params[0] << std::endl; - if (docs_params.size() > 1) { - auto path = std::filesystem::path(docs_params[1]); - if (std::filesystem::exists(path) && !std::filesystem::is_regular_file(path)) { - std::cerr << "PROCESSORS.md target path exists, but it is not a regular file: " << path << std::endl; - std::exit(1); - } - auto dir = path.parent_path(); - if (dir == docs_params[0]) { - std::cerr << "Target file should be out of the working directory: " << dir << std::endl; - std::exit(1); - } - std::ofstream outref(docs_params[1]); - dumpDocs(configure, docs_params[0], outref); - } else { - dumpDocs(configure, docs_params[0], std::cout); - } + dumpDocs(configure, docs_params[0]); std::exit(0); } @@ -209,10 +194,9 @@ int main(int argc, char **argv) { .metavar("KEY=VALUE") .help("Override a property read from minifi.properties file in key=value format"); argument_parser.add_argument("-d", "--docs") - .nargs(1, 2) - .metavar("PATH") - .help("Generate documentation in the directory specified in the first parameter. File path can be specified for the PROCESSORS.md file in the second parameter. " - "If no separate path is given for the PROCESSORS.md file, it will be printed to stdout."); + .nargs(1) + .metavar("DIRECTORY") + .help("Generate documentation in the specified directory"); argument_parser.add_argument("-s", "--schema") .metavar("PATH") .help("Generate JSON schema to the specified path"); From 0389e80d60dfc12519b793411203968d98f524d3 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Fri, 22 Sep 2023 16:48:22 +0200 Subject: [PATCH 2/5] use a raw string literal for the license string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Márton Szász --- minifi_main/AgentDocs.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index 91cfead48c..f9f4a9bb6d 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -73,20 +73,20 @@ std::string formatListOfRelationships(std::span"; +inline constexpr std::string_view APACHE_LICENSE = R"license()license"; void writeHeader(std::ostream& docs, const std::vector>& class_descriptions) { docs << APACHE_LICENSE; From 820e4ae455fc335cdd0594bf9495cc351c48e520 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Fri, 22 Sep 2023 17:10:37 +0200 Subject: [PATCH 3/5] use {} instead of = for initialization; VS likes this better --- libminifi/include/core/PropertyDefinition.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libminifi/include/core/PropertyDefinition.h b/libminifi/include/core/PropertyDefinition.h index e1a6d8a12c..12caa61e81 100644 --- a/libminifi/include/core/PropertyDefinition.h +++ b/libminifi/include/core/PropertyDefinition.h @@ -39,7 +39,7 @@ struct PropertyDefinition { std::array dependent_properties; std::array, NumExclusiveOfProperties> exclusive_of_properties; std::optional default_value; - gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; bool supports_expression_language = false; }; @@ -53,7 +53,7 @@ struct PropertyReference { std::span dependent_properties; std::span> exclusive_of_properties; std::optional default_value; - gsl::not_null type = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); + gsl::not_null type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; bool supports_expression_language = false; constexpr PropertyReference() = default; From 73c477c054b5336d63cb916a739d1875d61763b1 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Mon, 25 Sep 2023 09:54:59 +0200 Subject: [PATCH 4/5] change lowercaseFirst() to a function to match the other helper functions --- minifi_main/AgentDocs.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index f9f4a9bb6d..3257d997c6 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -164,7 +164,7 @@ std::string extractClassName(const std::string& full_class_name) { return minifi::utils::StringUtils::split(full_class_name, ".").back(); } -constexpr auto LowercaseFirst = [](const auto& key_value) { +std::string lowercaseFirst(const std::pair& key_value) { return minifi::utils::StringUtils::toLower(key_value.first); }; @@ -184,8 +184,8 @@ void AgentDocs::generate(const std::filesystem::path& docs_dir) { processors.emplace_back(extractClassName(processor_description.full_name_), processor_description); } } - ranges::sort(controller_services, std::less(), LowercaseFirst); - ranges::sort(processors, std::less(), LowercaseFirst); + ranges::sort(controller_services, std::less(), lowercaseFirst); + ranges::sort(processors, std::less(), lowercaseFirst); std::ofstream controllers_md(docs_dir / "CONTROLLERS.md"); writeHeader(controllers_md, controller_services); From 7af96ad66e66762ce12a3a199bfc2d0844b3fe82 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Mon, 25 Sep 2023 14:32:19 +0200 Subject: [PATCH 5/5] remove the bugprone-unchecked-optional-access clang-tidy check --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 2345a2425a..ac7598997a 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -55,7 +55,6 @@ Checks: > bugprone-terminating-continue, bugprone-throw-keyword-missing, bugprone-too-small-loop-variable, - bugprone-unchecked-optional-access, bugprone-undefined-memory-manipulation, bugprone-undelegated-constructor, bugprone-unhandled-exception-at-new,