From 92ccb79670fd5990d915cb3d5c7b7eb0bb182b52 Mon Sep 17 00:00:00 2001 From: krisztina-zsihovszki Date: Tue, 31 Jan 2023 17:38:19 +0100 Subject: [PATCH] NIFI-11117 Remove folder creation from PutGoogleDrive This closes #6910. Signed-off-by: Peter Turcsanyi --- .../nifi-gcp-processors/pom.xml | 5 + .../gcp/drive/FetchGoogleDrive.java | 4 +- .../processors/gcp/drive/PutGoogleDrive.java | 123 +++--------------- .../gcp/drive/FetchGoogleDriveIT.java | 11 -- .../gcp/drive/ListGoogleDriveIT.java | 15 +-- .../gcp/drive/ListGoogleDriveSimpleTest.java | 1 - .../drive/ListGoogleDriveTestRunnerTest.java | 14 -- .../gcp/drive/PutGoogleDriveIT.java | 68 +--------- .../gcp/drive/PutGoogleDriveTest.java | 59 ++------- 9 files changed, 48 insertions(+), 252 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml index 2063da4b543e..0517d7023fe8 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml @@ -65,6 +65,11 @@ 1.20.0-SNAPSHOT provided + + org.apache.nifi + nifi-conflict-resolution + 1.20.0-SNAPSHOT + org.apache.nifi nifi-mock diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDrive.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDrive.java index c958cd448987..40ae075de16b 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDrive.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDrive.java @@ -183,7 +183,7 @@ private File fetchFileMetadata(String fileId) throws IOException { } private void handleErrorResponse(ProcessSession session, String fileId, FlowFile flowFile, GoogleJsonResponseException e) { - getLogger().error("Couldn't fetch file with id '{}'", fileId, e); + getLogger().error("Fetching File [{}] failed", fileId, e); flowFile = session.putAttribute(flowFile, ERROR_CODE, "" + e.getStatusCode()); flowFile = session.putAttribute(flowFile, ERROR_MESSAGE, e.getMessage()); @@ -193,7 +193,7 @@ private void handleErrorResponse(ProcessSession session, String fileId, FlowFile } private void handleUnexpectedError(ProcessSession session, FlowFile flowFile, String fileId, Exception e) { - getLogger().error("Unexpected error while fetching and processing file with id '{}'", fileId, e); + getLogger().error("Fetching File [{}] failed", fileId, e); flowFile = session.putAttribute(flowFile, ERROR_MESSAGE, e.getMessage()); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/PutGoogleDrive.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/PutGoogleDrive.java index 0276c3a0eb3c..751a6397fbcf 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/PutGoogleDrive.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/drive/PutGoogleDrive.java @@ -23,7 +23,8 @@ import static java.util.Collections.singletonList; import static java.util.stream.Collectors.joining; import static org.apache.nifi.processor.util.StandardValidators.DATA_SIZE_VALIDATOR; -import static org.apache.nifi.processor.util.StandardValidators.createRegexMatchingValidator; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.FAIL; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.IGNORE; import static org.apache.nifi.processors.gcp.drive.GoogleDriveAttributes.ERROR_CODE; import static org.apache.nifi.processors.gcp.drive.GoogleDriveAttributes.ERROR_CODE_DESC; import static org.apache.nifi.processors.gcp.drive.GoogleDriveAttributes.ERROR_MESSAGE; @@ -66,7 +67,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.behavior.ReadsAttribute; @@ -90,6 +90,7 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy; import org.apache.nifi.processors.gcp.ProxyAwareTransportFactory; import org.apache.nifi.proxy.ProxyConfiguration; import org.json.JSONObject; @@ -108,34 +109,19 @@ @WritesAttribute(attribute = ERROR_CODE, description = ERROR_CODE_DESC), @WritesAttribute(attribute = ERROR_MESSAGE, description = ERROR_MESSAGE_DESC)}) public class PutGoogleDrive extends AbstractProcessor implements GoogleDriveTrait { - - public static final String IGNORE_RESOLUTION = "ignore"; - public static final String REPLACE_RESOLUTION = "replace"; - public static final String FAIL_RESOLUTION = "fail"; public static final int MIN_ALLOWED_CHUNK_SIZE_IN_BYTES = MediaHttpUploader.MINIMUM_CHUNK_SIZE; public static final int MAX_ALLOWED_CHUNK_SIZE_IN_BYTES = 1024 * 1024 * 1024; public static final PropertyDescriptor FOLDER_ID = new PropertyDescriptor.Builder() .name("folder-id") .displayName("Folder ID") - .description("The ID of the shared folder. " + + .description("The ID of the shared folder." + " Please see Additional Details to set up access to Google Drive and obtain Folder ID.") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .required(true) .build(); - public static final PropertyDescriptor SUBFOLDER_NAME = new PropertyDescriptor.Builder() - .name("subfolder-name") - .displayName("Subfolder Name") - .description("The name (path) of the subfolder where files are uploaded. The subfolder name is relative to the shared folder specified by 'Folder ID'." - + " Example: subFolder, subFolder1/subfolder2") - .addValidator(createRegexMatchingValidator(Pattern.compile("^(?!/).+(? PROPERTIES = Collections.unmodifiableList(asList( GCP_CREDENTIALS_PROVIDER_SERVICE, FOLDER_ID, - SUBFOLDER_NAME, - CREATE_SUBFOLDER, FILE_NAME, CONFLICT_RESOLUTION, CHUNKED_UPLOAD_THRESHOLD, @@ -260,15 +231,11 @@ public void onTrigger(final ProcessContext context, final ProcessSession session return; } - String folderId = context.getProperty(FOLDER_ID).evaluateAttributeExpressions(flowFile).getValue(); - final String subfolderName = context.getProperty(SUBFOLDER_NAME).evaluateAttributeExpressions(flowFile).getValue(); - final boolean createFolder = context.getProperty(CREATE_SUBFOLDER).asBoolean(); + final String folderId = context.getProperty(FOLDER_ID).evaluateAttributeExpressions(flowFile).getValue(); final String filename = context.getProperty(FILE_NAME).evaluateAttributeExpressions(flowFile).getValue(); final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()); try { - folderId = subfolderName != null ? getOrCreateParentSubfolder(subfolderName, folderId, createFolder).getId() : folderId; - final long startNanos = System.nanoTime(); final long size = flowFile.getSize(); @@ -280,20 +247,19 @@ public void onTrigger(final ProcessContext context, final ProcessSession session .asDataSize(DataUnit.B) .intValue(); - final String conflictResolution = context.getProperty(CONFLICT_RESOLUTION).getValue(); - + final ConflictResolutionStrategy conflictResolution = ConflictResolutionStrategy.forValue(context.getProperty(CONFLICT_RESOLUTION).getValue()); final Optional alreadyExistingFile = checkFileExistence(filename, folderId); - final File fileMetadata = alreadyExistingFile.isPresent() ? alreadyExistingFile.get() : createMetadata(filename, folderId); + final File fileMetadata = alreadyExistingFile.orElseGet(() -> createMetadata(filename, folderId)); - if (alreadyExistingFile.isPresent() && FAIL_RESOLUTION.equals(conflictResolution)) { - getLogger().error("File '{}' already exists in {} folder, conflict resolution is '{}'", filename, getFolderName(subfolderName), FAIL_RESOLUTION); + if (alreadyExistingFile.isPresent() && conflictResolution == FAIL) { + getLogger().error("File [{}] already exists in [{}] Folder, conflict resolution is [{}]", filename, folderId, FAIL.getDisplayName()); flowFile = addAttributes(alreadyExistingFile.get(), flowFile, session); session.transfer(flowFile, REL_FAILURE); return; } - if (alreadyExistingFile.isPresent() && IGNORE_RESOLUTION.equals(conflictResolution)) { - getLogger().info("File '{}' already exists in {} folder, conflict resolution is '{}'", filename, getFolderName(subfolderName), IGNORE_RESOLUTION); + if (alreadyExistingFile.isPresent() && conflictResolution == IGNORE) { + getLogger().info("File [{}] already exists in [{}] Folder, conflict resolution is [{}]", filename, folderId, IGNORE.getDisplayName()); flowFile = addAttributes(alreadyExistingFile.get(), flowFile, session); session.transfer(flowFile, REL_SUCCESS); return; @@ -323,12 +289,12 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } session.transfer(flowFile, REL_SUCCESS); } catch (GoogleJsonResponseException e) { - getLogger().error("Exception occurred while uploading file '{}' to {} Google Drive folder", filename, - getFolderName(subfolderName), e); + getLogger().error("Exception occurred while uploading File [{}] to [{}] Google Drive Folder", filename, + folderId, e); handleExpectedError(session, flowFile, e); } catch (Exception e) { - getLogger().error("Exception occurred while uploading file '{}' to {} Google Drive folder", filename, - getFolderName(subfolderName), e); + getLogger().error("Exception occurred while uploading File [{}] to [{}] Google Drive Folder", filename, + folderId, e); if (e.getCause() != null && e.getCause() instanceof GoogleJsonResponseException) { handleExpectedError(session, flowFile, (GoogleJsonResponseException) e.getCause()); @@ -354,10 +320,6 @@ private FlowFile addAttributes(File file, FlowFile flowFile, ProcessSession sess return session.putAllAttributes(flowFile, attributes); } - private String getFolderName(String subFolderName) { - return subFolderName == null ? "shared" : format("'%s'", subFolderName); - } - private DriveRequest createDriveRequest(File fileMetadata, final InputStreamContent mediaContent) throws IOException { if (fileMetadata.getId() == null) { return driveService.files() @@ -384,7 +346,8 @@ private File uploadFileInChunks(DriveRequest driveRequest, File fileMetada fileMetadata.setSize(mediaContent.getLength()); return fileMetadata; } else { - throw new ProcessException(format("Upload of file '%s' to folder '%s' failed, HTTP error code: %d", fileMetadata.getName(), fileMetadata.getId(), response.getStatusCode())); + throw new ProcessException(format("Upload of File [%s] to Folder [%s] failed, HTTP error code: [%d]", + fileMetadata.getName(), fileMetadata.getId(), response.getStatusCode())); } } @@ -395,44 +358,6 @@ private String getUploadedFileId(final InputStream content) { return new JSONObject(contentAsString).getString("id"); } - private File getOrCreateParentSubfolder(String folderName, String parentFolderId, boolean createFolder) throws IOException { - final int indexOfPathSeparator = folderName.indexOf("/"); - - if (isMultiLevelFolder(indexOfPathSeparator, folderName)) { - final String mainFolderName = folderName.substring(0, indexOfPathSeparator); - final String subFolders = folderName.substring(indexOfPathSeparator + 1); - final File mainFolder = getOrCreateFolder(mainFolderName, parentFolderId, createFolder); - return getOrCreateParentSubfolder(subFolders, mainFolder.getId(), createFolder); - } else { - return getOrCreateFolder(folderName, parentFolderId, createFolder); - } - } - - private boolean isMultiLevelFolder(int indexOfPathSeparator, String folderName) { - return indexOfPathSeparator > 0 && indexOfPathSeparator < folderName.length() - 1; - } - - private File getOrCreateFolder(String folderName, String parentFolderId, boolean createFolder) throws IOException { - final Optional existingFolder = checkFolderExistence(folderName, parentFolderId); - - if (existingFolder.isPresent()) { - return existingFolder.get(); - } - - if (createFolder) { - getLogger().debug("Create folder " + folderName + " parent id: " + parentFolderId); - final File folderMetadata = createMetadata(folderName, parentFolderId); - folderMetadata.setMimeType(DRIVE_FOLDER_MIME_TYPE); - - return driveService.files() - .create(folderMetadata) - .setFields("id, parents") - .execute(); - } else { - throw new ProcessException(format("The specified subfolder '%s' does not exist and '%s' is false.", folderName, CREATE_SUBFOLDER.getDisplayName())); - } - } - private File createMetadata(final String name, final String parentId) { final File metadata = new File(); metadata.setName(name); @@ -440,18 +365,10 @@ private File createMetadata(final String name, final String parentId) { return metadata; } - private Optional checkFolderExistence(String folderName, String parentId) throws IOException { - return checkObjectExistence(format("mimeType='%s' and name='%s' and ('%s' in parents)", DRIVE_FOLDER_MIME_TYPE, folderName, parentId)); - } - private Optional checkFileExistence(String fileName, String parentId) throws IOException { - return checkObjectExistence(format("name='%s' and ('%s' in parents)", fileName, parentId)); - } - - private Optional checkObjectExistence(String query) throws IOException { final FileList result = driveService.files() .list() - .setQ(query) + .setQ(format("name='%s' and ('%s' in parents)", fileName, parentId)) .setFields("files(name, id)") .execute(); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDriveIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDriveIT.java index 8b10406fc795..9220b4d02e24 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDriveIT.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/FetchGoogleDriveIT.java @@ -42,7 +42,6 @@ public FetchGoogleDrive createTestSubject() { @Test void testFetch() throws Exception { - // GIVEN File file = createFileWithDefaultContent("test_file.txt", mainFolderId); Map inputFlowFileAttributes = new HashMap<>(); @@ -54,11 +53,9 @@ void testFetch() throws Exception { HashSet> expectedAttributes = new HashSet<>(singletonList(inputFlowFileAttributes)); List expectedContent = singletonList(DEFAULT_FILE_CONTENT); - // WHEN testRunner.enqueue("unimportant_data", inputFlowFileAttributes); testRunner.run(); - // THEN testRunner.assertTransferCount(FetchGoogleDrive.REL_FAILURE, 0); checkAttributes(FetchGoogleDrive.REL_SUCCESS, expectedAttributes); @@ -67,7 +64,6 @@ void testFetch() throws Exception { @Test void testInputFlowFileReferencesMissingFile() { - // GIVEN Map inputFlowFileAttributes = new HashMap<>(); inputFlowFileAttributes.put(GoogleDriveAttributes.ID, "missing"); inputFlowFileAttributes.put(GoogleDriveAttributes.FILENAME, "missing_filename"); @@ -80,24 +76,20 @@ void testInputFlowFileReferencesMissingFile() { }} )); - // WHEN testRunner.enqueue("unimportant_data", inputFlowFileAttributes); testRunner.run(); - // THEN testRunner.assertTransferCount(FetchGoogleDrive.REL_SUCCESS, 0); checkAttributes(FetchGoogleDrive.REL_FAILURE, expectedFailureAttributes); } @Test void testInputFlowFileThrowsExceptionBeforeFetching() throws Exception { - // GIVEN File file = createFileWithDefaultContent("test_file.txt", mainFolderId); Map inputFlowFileAttributes = new HashMap<>(); inputFlowFileAttributes.put(GoogleDriveAttributes.ID, file.getId()); inputFlowFileAttributes.put(GoogleDriveAttributes.FILENAME, file.getName()); - MockFlowFile input = new MockFlowFile(1) { final AtomicBoolean throwException = new AtomicBoolean(true); @@ -111,7 +103,6 @@ public boolean isPenalized() { return super.isPenalized(); } } - @Override public Map getAttributes() { return inputFlowFileAttributes; @@ -122,11 +113,9 @@ public Map getAttributes() { inputFlowFileAttributes )); - // WHEN testRunner.enqueue(input); testRunner.run(); - // THEN testRunner.assertTransferCount(FetchGoogleDrive.REL_SUCCESS, 0); checkAttributes(FetchGoogleDrive.REL_FAILURE, expectedFailureAttributes); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveIT.java index f352539064c4..ed7fab0a0871 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveIT.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveIT.java @@ -49,7 +49,6 @@ public ListGoogleDrive createTestSubject() { @Test void listFilesFrom3LayerDeepDirectoryTree() throws Exception { - // GIVEN File main_sub1 = createFolder("main_sub1", mainFolderId); File main_sub2 = createFolder("main_sub2", mainFolderId); @@ -78,10 +77,8 @@ void listFilesFrom3LayerDeepDirectoryTree() throws Exception { // The creation of the files are not (completely) synchronized. Thread.sleep(2000); - // WHEN testRunner.run(); - // THEN List successFlowFiles = testRunner.getFlowFilesForRelationship(ListGoogleDrive.REL_SUCCESS); Set actualFileNames = successFlowFiles.stream() @@ -93,19 +90,17 @@ void listFilesFrom3LayerDeepDirectoryTree() throws Exception { // Next, list a sub folder, non-recursively this time. (Changing these properties will clear the Processor state as well // so all files are eligible for listing again.) - // GIVEN + testRunner.clearTransferState(); expectedFileNames = new HashSet<>(Arrays.asList( "main_sub1_file1" )); - // WHEN testRunner.setProperty(ListGoogleDrive.FOLDER_ID, main_sub1.getId()); testRunner.setProperty(ListGoogleDrive.RECURSIVE_SEARCH, "false"); testRunner.run(); - // THEN successFlowFiles = testRunner.getFlowFilesForRelationship(ListGoogleDrive.REL_SUCCESS); actualFileNames = successFlowFiles.stream() @@ -117,7 +112,6 @@ void listFilesFrom3LayerDeepDirectoryTree() throws Exception { @Test void doNotListTooYoungFilesWhenMinAgeIsSet() throws Exception { - // GIVEN testRunner.setProperty(ListGoogleDrive.MIN_AGE, "15 s"); createFileWithDefaultContent("main_file", mainFolderId); @@ -125,10 +119,8 @@ void doNotListTooYoungFilesWhenMinAgeIsSet() throws Exception { // Make sure the file 'arrives' and could be listed Thread.sleep(5000); - // WHEN testRunner.run(); - // THEN List successFlowFiles = testRunner.getFlowFilesForRelationship(ListGoogleDrive.REL_SUCCESS); Set actualFileNames = successFlowFiles.stream() @@ -139,17 +131,15 @@ void doNotListTooYoungFilesWhenMinAgeIsSet() throws Exception { // Next, wait for another 10+ seconds for MIN_AGE to expire then list again - // GIVEN + Thread.sleep(10000); Set expectedFileNames = new HashSet<>(Arrays.asList( "main_file" )); - // WHEN testRunner.run(); - // THEN successFlowFiles = testRunner.getFlowFilesForRelationship(ListGoogleDrive.REL_SUCCESS); actualFileNames = successFlowFiles.stream() @@ -158,5 +148,4 @@ void doNotListTooYoungFilesWhenMinAgeIsSet() throws Exception { assertEquals(expectedFileNames, actualFileNames); } - } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveSimpleTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveSimpleTest.java index e29fc62a57b9..47394a258897 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveSimpleTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveSimpleTest.java @@ -38,7 +38,6 @@ public class ListGoogleDriveSimpleTest { private ListGoogleDrive testSubject; - private ProcessContext mockProcessContext; private Drive mockDriverService; diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveTestRunnerTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveTestRunnerTest.java index af3d324005c3..da628c25151d 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveTestRunnerTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/ListGoogleDriveTestRunnerTest.java @@ -84,7 +84,6 @@ public Drive createDriveService(ProcessContext context, HttpTransport httpTransp @Test void testOutputAsAttributesWhereTimestampIsCreatedTime() throws Exception { - // GIVEN String id = "id_1"; String filename = "file_name_1"; Long size = 125L; @@ -92,14 +91,10 @@ void testOutputAsAttributesWhereTimestampIsCreatedTime() throws Exception { Long modifiedTime = null; String mimeType = "mime_type_1"; - // WHEN - // THEN testOutputAsAttributes(id, filename, size, createdTime, modifiedTime, mimeType, createdTime); } - @Test void testOutputAsAttributesWhereTimestampIsModifiedTime() throws Exception { - // GIVEN String id = "id_1"; String filename = "file_name_1"; Long size = 125L; @@ -107,14 +102,11 @@ void testOutputAsAttributesWhereTimestampIsModifiedTime() throws Exception { Long modifiedTime = 123456L + 1L; String mimeType = "mime_type_1"; - // WHEN - // THEN testOutputAsAttributes(id, filename, size, createdTime, modifiedTime, mimeType, modifiedTime); } @Test void testOutputAsContent() throws Exception { - // GIVEN String id = "id_1"; String filename = "file_name_1"; Long size = 125L; @@ -137,10 +129,7 @@ void testOutputAsContent() throws Exception { "}" + "]"); - // WHEN testRunner.run(); - - // THEN checkContent(ListGoogleDrive.REL_SUCCESS, expectedContents); } @@ -172,7 +161,6 @@ private void mockFetchedGoogleDriveFileList(String id, String filename, Long siz } private void testOutputAsAttributes(String id, String filename, Long size, Long createdTime, Long modifiedTime, String mimeType, Long expectedTimestamp) throws IOException { - // GIVEN mockFetchedGoogleDriveFileList(id, filename, size, createdTime, modifiedTime, mimeType); Map inputFlowFileAttributes = new HashMap<>(); @@ -184,10 +172,8 @@ private void testOutputAsAttributes(String id, String filename, Long size, Long HashSet> expectedAttributes = new HashSet<>(asList(inputFlowFileAttributes)); - // WHEN testRunner.run(); - // THEN checkAttributes(ListGoogleDrive.REL_SUCCESS, expectedAttributes); } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveIT.java index 98ac34799213..5ee3dd550fb9 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveIT.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveIT.java @@ -16,12 +16,11 @@ */ package org.apache.nifi.processors.gcp.drive; -import static org.apache.nifi.processors.gcp.drive.PutGoogleDrive.CREATE_SUBFOLDER; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.IGNORE; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.REPLACE; import static org.apache.nifi.processors.gcp.drive.PutGoogleDrive.FILE_NAME; import static org.apache.nifi.processors.gcp.drive.PutGoogleDrive.FOLDER_ID; -import static org.apache.nifi.processors.gcp.drive.PutGoogleDrive.SUBFOLDER_NAME; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,68 +58,11 @@ void testUploadFileToFolderById() { // THEN testRunner.assertTransferCount(PutGoogleDrive.REL_SUCCESS, 1); testRunner.assertTransferCount(PutGoogleDrive.REL_FAILURE, 0); - } - - @Test - void testUploadFileFolderByName() { - // GIVEN - testRunner.setProperty(SUBFOLDER_NAME, "testFolderNew"); - testRunner.setProperty(FOLDER_ID, mainFolderId); - testRunner.setProperty(FILE_NAME, TEST_FILENAME); - testRunner.setProperty(CREATE_SUBFOLDER, "true"); - - // WHEN - runWithFileContent(); - - // THEN - testRunner.assertTransferCount(PutGoogleDrive.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutGoogleDrive.REL_FAILURE, 0); - final List flowFiles = testRunner.getFlowFilesForRelationship(PutGoogleDrive.REL_SUCCESS); - final MockFlowFile ff0 = flowFiles.get(0); - assertFlowFileAttributes(ff0); - } - - @Test - void testUploadFileCreateMultiLevelFolder() throws IOException { - createFolder("existingFolder", mainFolderId); - - // GIVEN - testRunner.setProperty(SUBFOLDER_NAME, "existingFolder/new1/new2"); - testRunner.setProperty(FOLDER_ID, mainFolderId); - testRunner.setProperty(FILE_NAME, TEST_FILENAME); - testRunner.setProperty(CREATE_SUBFOLDER, "true"); - - // WHEN - runWithFileContent(); - - // THEN - testRunner.assertTransferCount(PutGoogleDrive.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutGoogleDrive.REL_FAILURE, 0); - final List flowFiles = testRunner.getFlowFilesForRelationship(PutGoogleDrive.REL_SUCCESS); final MockFlowFile ff0 = flowFiles.get(0); assertFlowFileAttributes(ff0); } - @Test - void testSpecifiedFolderIdDoesNotExist() { - // GIVEN - testRunner.setProperty(FOLDER_ID, "nonExistentId"); - testRunner.setProperty(FILE_NAME, "testFile4"); - - // WHEN - runWithFileContent(); - - // THEN - testRunner.assertTransferCount(PutGoogleDrive.REL_SUCCESS, 0); - testRunner.assertTransferCount(PutGoogleDrive.REL_FAILURE, 1); - - final List flowFiles = testRunner.getFlowFilesForRelationship(PutGoogleDrive.REL_FAILURE); - final MockFlowFile ff0 = flowFiles.get(0); - ff0.assertAttributeEquals(GoogleDriveAttributes.ERROR_CODE, "404"); - ff0.assertAttributeExists(GoogleDriveAttributes.ERROR_MESSAGE); - } - @Test void testUploadedFileAlreadyExistsFailResolution() { // GIVEN @@ -145,11 +87,11 @@ void testUploadedFileAlreadyExistsFailResolution() { } @Test - void testUploadedFileAlreadyExistsOverwriteResolution() { + void testUploadedFileAlreadyExistsReplaceResolution() { // GIVEN testRunner.setProperty(FOLDER_ID, mainFolderId); testRunner.setProperty(FILE_NAME, TEST_FILENAME); - testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, PutGoogleDrive.REPLACE_RESOLUTION); + testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, REPLACE.getValue()); // WHEN runWithFileContent(); @@ -176,7 +118,7 @@ void testUploadedFileAlreadyExistsIgnoreResolution() { // GIVEN testRunner.setProperty(FOLDER_ID, mainFolderId); testRunner.setProperty(FILE_NAME, TEST_FILENAME); - testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, PutGoogleDrive.IGNORE_RESOLUTION); + testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, IGNORE.getValue()); // WHEN runWithFileContent(); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveTest.java index b9e45e093d60..1e1905ac62f2 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/drive/PutGoogleDriveTest.java @@ -19,9 +19,11 @@ import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.IGNORE; +import static org.apache.nifi.processors.conflict.resolution.ConflictResolutionStrategy.REPLACE; import static org.apache.nifi.processors.gcp.drive.GoogleDriveAttributes.ERROR_MESSAGE; -import static org.apache.nifi.processors.gcp.drive.GoogleDriveTrait.DRIVE_FOLDER_MIME_TYPE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -82,26 +84,11 @@ void testUploadChunkSizeValidity() { testRunner.assertNotValid(); } - @Test - void testSubfolderNameValidity() { - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "sub1"); - testRunner.assertValid(); - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "sub1/sub2"); - testRunner.assertValid(); - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "/sub1"); - testRunner.assertNotValid(); - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "/"); - testRunner.assertNotValid(); - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "sub1/"); - testRunner.assertNotValid(); - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, "/sub1/"); - testRunner.assertNotValid(); - } - @Test void testFileUploadFileNameFromProperty() throws Exception { testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); + mockFileExists(emptyList()); mockFileUpload(createFile()); runWithFlowFile(); @@ -114,6 +101,7 @@ void testFileUploadFileNameFromProperty() throws Exception { void testFileUploadFileNameFromFlowFileAttribute() throws Exception { testRunner.setProperty(PutGoogleDrive.FOLDER_ID, SHARED_FOLDER_ID); + mockFileExists(emptyList()); mockFileUpload(createFile()); final MockFlowFile mockFlowFile = getMockFlowFile(); @@ -128,31 +116,12 @@ void testFileUploadFileNameFromFlowFileAttribute() throws Exception { assertProvenanceEvent(ProvenanceEventType.SEND); } - @Test - void testFileUploadFileToFolderSpecifiedByNameFolderExists() throws Exception { - testRunner.setProperty(PutGoogleDrive.SUBFOLDER_NAME, SUBFOLDER_NAME); - testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); - - when(mockDriverService.files() - .list() - .setQ(format("mimeType='%s' and name='%s' and ('%s' in parents)", DRIVE_FOLDER_MIME_TYPE, SUBFOLDER_NAME, SHARED_FOLDER_ID)) - .setFields("files(name, id)") - .execute()) - .thenReturn(new FileList().setFiles(singletonList(createFile(SUBFOLDER_ID, SUBFOLDER_NAME, SHARED_FOLDER_ID, DRIVE_FOLDER_MIME_TYPE)))); - - mockFileUpload(createFile()); - - runWithFlowFile(); - testRunner.assertAllFlowFilesTransferred(PutGoogleDrive.REL_SUCCESS, 1); - assertFlowFileAttributes(PutGoogleDrive.REL_SUCCESS); - assertProvenanceEvent(ProvenanceEventType.SEND); - } - @Test void testFileUploadError() throws Exception { testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); final JsonParseException exception = new JsonParseException("Google Drive error", new FileNotFoundException()); + mockFileExists(emptyList()); mockFileUploadError(exception); runWithFlowFile(); @@ -168,7 +137,7 @@ void testFileUploadError() throws Exception { void testFileAlreadyExistsFailResolution() throws Exception { testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); - mockFileExists(); + mockFileExists(singletonList(createFile())); runWithFlowFile(); @@ -179,9 +148,9 @@ void testFileAlreadyExistsFailResolution() throws Exception { @Test void testFileAlreadyExistsIgnoreResolution() throws Exception { testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); - testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, PutGoogleDrive.IGNORE_RESOLUTION); + testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, IGNORE.getValue()); - mockFileExists(); + mockFileExists(singletonList(createFile())); runWithFlowFile(); @@ -193,11 +162,11 @@ void testFileAlreadyExistsIgnoreResolution() throws Exception { } @Test - void testFileAlreadyExistsOverwriteResolution() throws Exception { + void testFileAlreadyExistsReplaceResolution() throws Exception { testRunner.setProperty(PutGoogleDrive.FILE_NAME, TEST_FILENAME); - testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, PutGoogleDrive.REPLACE_RESOLUTION); + testRunner.setProperty(PutGoogleDrive.CONFLICT_RESOLUTION, REPLACE.getValue()); - mockFileExists(); + mockFileExists(singletonList(createFile())); mockFileUpdate(createFile()); @@ -242,12 +211,12 @@ private void mockFileUploadError(Exception exception) throws IOException { .thenThrow(exception); } - private void mockFileExists() throws IOException { + private void mockFileExists(List fileList) throws IOException { when(mockDriverService.files() .list() .setQ(format("name='%s' and ('%s' in parents)", TEST_FILENAME, SHARED_FOLDER_ID)) .setFields("files(name, id)") .execute()) - .thenReturn(new FileList().setFiles(singletonList(createFile()))); + .thenReturn(new FileList().setFiles(fileList)); } } \ No newline at end of file