Add config options for PK chunking

forcedotcom · Apr 16, 2018 · 23f4582 · 23f4582
1 parent 13582dc
commit 23f4582
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 0 deletions.
diff --git a/src/main/java/com/salesforce/dataloader/config/Config.java b/src/main/java/com/salesforce/dataloader/config/Config.java
@@ -87,6 +87,8 @@ public class Config {
     public static final int DEFAULT_BULK_API_BATCH_SIZE = 2000;
     public static final long DEFAULT_BULK_API_CHECK_STATUS_INTERVAL = 5000L;
     public static final String DEFAULT_ENDPOINT_URL = "https://login.salesforce.com";
+    public static final int DEFAULT_BULK_QUERY_CHUNK_SIZE = 100000;
+    public static final int MAX_BULK_QUERY_CHUNK_SIZE = 250000;
 
     /*
      * Issue #59 - Dataloader will not read all the database rows to get a total count
@@ -161,6 +163,8 @@ public class Config {
     public static final String BULK_API_SERIAL_MODE = "sfdc.bulkApiSerialMode";
     public static final String BULK_API_CHECK_STATUS_INTERVAL = "sfdc.bulkApiCheckStatusInterval";
     public static final String BULK_API_ZIP_CONTENT = "sfdc.bulkApiZipContent";
+    public static final String BULK_QUERY_PK_CHUNKING = "sfdc.bulkQueryPKChunking";
+    public static final String BULK_QUERY_CHUNK_SIZE = "sfcd.bulkQueryChunkSize";
     public static final String WIRE_OUTPUT = "sfdc.wireOutput";
     public static final String TIMEZONE = "sfdc.timezone";
 
@@ -342,6 +346,8 @@ public void setDefaults() {
         setValue(BULK_API_SERIAL_MODE, false);
         setValue(BULK_API_ZIP_CONTENT, false);
         setValue(BULK_API_CHECK_STATUS_INTERVAL, DEFAULT_BULK_API_CHECK_STATUS_INTERVAL);
+        setValue(BULK_QUERY_PK_CHUNKING, false);
+        setValue(BULK_QUERY_CHUNK_SIZE, DEFAULT_BULK_QUERY_CHUNK_SIZE);
         setValue(WIRE_OUTPUT, false);
         setValue(TIMEZONE, TimeZone.getDefault().getID());
         //sfdcInternal settings

diff --git a/src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java b/src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java
@@ -97,6 +97,8 @@ public class AdvancedSettingsDialog extends Dialog {
     private Button buttonCsvComma;
     private Button buttonCsvTab;
     private Button buttonCsvOther;
+    private Button buttonBulkQueryPKChunking;
+    private Text textBulkQueryChunkSize;
 
     /**
      * InputDialog constructor
@@ -499,6 +501,30 @@ public void widgetSelected(SelectionEvent e) {
         buttonBulkApiZipContent = new Button(restComp, SWT.CHECK);
         buttonBulkApiZipContent.setSelection(config.getBoolean(Config.BULK_API_SERIAL_MODE));
         buttonBulkApiZipContent.setEnabled(useBulkAPI);
+
+        Label labelBulkQueryPKChunking = new Label(restComp, SWT.RIGHT);
+        labelBulkQueryPKChunking.setText(Labels.getString("AdvancedSettingsDialog.bulkQueryPKChunking"));
+        labelBulkQueryPKChunking.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_END));
+
+        buttonBulkQueryPKChunking = new Button(restComp, SWT.CHECK);
+        buttonBulkQueryPKChunking.setSelection(config.getBoolean(Config.BULK_QUERY_PK_CHUNKING));
+        buttonBulkQueryPKChunking.setEnabled(useBulkAPI);
+
+        Label labelBulkQueryChunkSize = new Label(restComp, SWT.RIGHT);
+        labelBulkQueryChunkSize.setText(Labels.getString("AdvancedSettingsDialog.bulkQueryChunkSize"));
+        labelBulkQueryChunkSize.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_END));
+
+        textBulkQueryChunkSize = new Text(restComp, SWT.BORDER);
+        textBulkQueryChunkSize.setText(config.getString(Config.BULK_QUERY_CHUNK_SIZE));
+        textBulkQueryChunkSize.setTextLimit(6);
+        textBulkQueryChunkSize.addVerifyListener(new VerifyListener() {
+            @Override
+            public void verifyText(VerifyEvent event) {
+                event.doit = Character.isISOControl(event.character) || Character.isDigit(event.character);
+            }
+        });
+        textBulkQueryChunkSize.setEnabled(useBulkAPI);
+
         // timezone
         textTimezone = createTextInput(restComp, "AdvancedSettingsDialog.timezone", Config.TIMEZONE, TimeZone.getDefault().getID(), 200);
 
@@ -686,6 +712,8 @@ public void widgetSelected(SelectionEvent event) {
                 config.setValue(Config.BULK_API_ENABLED, buttonUseBulkApi.getSelection());
                 config.setValue(Config.BULK_API_SERIAL_MODE, buttonBulkApiSerialMode.getSelection());
                 config.setValue(Config.BULK_API_ZIP_CONTENT, buttonBulkApiZipContent.getSelection());
+                config.setValue(Config.BULK_QUERY_PK_CHUNKING, buttonBulkQueryPKChunking.getSelection());
+                config.setValue(Config.BULK_QUERY_CHUNK_SIZE, textBulkQueryChunkSize.getText());
 
                 controller.saveConfig();
                 controller.logout();

diff --git a/src/main/resources/labels.properties b/src/main/resources/labels.properties
@@ -97,6 +97,8 @@ AdvancedSettingsDialog.useCommaAsCsvDelimiter=Allow comma as a CSV delimiter:
 AdvancedSettingsDialog.useTabAsCsvDelimiter=Allow Tab as a CSV delimiter:
 AdvancedSettingsDialog.useOtherAsCsvDelimiter=Allow other characters as CSV delimiters:
 AdvancedSettingsDialog.csvOtherDelimiterValue=Other Delimiters (enter multiple values with no separator; for example, !+?):
+AdvancedSettingsDialog.bulkQueryPKChunking=Enable PK Chunking for bulk query:
+AdvancedSettingsDialog.bulkQueryChunkSize=Query chunk size:
 
 InsertWizard.title=Load Inserts
 InsertWizard.confFirstLine=You have chosen to insert new records.  Click Yes to begin.