Merge pull request #1436 from ashitsalesforce/master

test for ability to create multiple bulk v2 jobs

src/main/java/com/salesforce/dataloader/action/AbstractLoadAction.java

@@ -79,7 +79,7 @@ protected void checkDao(DataAccessObject dao) throws DataAccessObjectInitializat
     protected boolean visit() throws DataAccessObjectException, ParameterLoadException, OperationException,
     ConnectionException {
 
-        final int loadBatchSize = this.getConfig().getMaxRowsInImportBatch();
+        final int loadBatchSize = this.getConfig().getCurrentSettingForMaxRowsInImportBatch();
         final int daoRowNumBase = getDao().getCurrentRowNumber();
         final List<TableRow> daoRowList = getDao().readTableRowList(loadBatchSize);
         if (daoRowList == null || daoRowList.size() == 0) return false;
@@ -101,6 +101,7 @@ protected boolean visit() throws DataAccessObjectException, ParameterLoadExcepti
                         // BulkV2 completed a job.
                         // Create a new visitor for a new job. However, it should use
                         // the LoadRateCalculator instance from the current visitor.
+                        getLogger().info("Need to run multiple Bulkv2 API jobs to complete upload");
                         setVisitor(this.createVisitor(this.getVisitor().getLoadRateCalculator(), false));
                     }
                     successfulVisit = getVisitor().visit(daoRow);

src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java

@@ -112,7 +112,7 @@ protected DAOLoadVisitor(Controller controller, ILoaderProgress monitor, DataWri
         dynaArray = dynaList;
         SforceDynaBean.registerConverters(getConfig());
 
-        this.MAX_ROWS_IN_BATCH = getConfig().getMaxRowsInImportBatch();
+        this.MAX_ROWS_IN_BATCH = getConfig().getCurrentSettingForMaxRowsInImportBatch();
         rowConversionFailureMap = new HashMap<Integer, Boolean>();
         String newRichTextRegex = getConfig().getString(AppConfig.PROP_RICH_TEXT_FIELD_REGEX);
         if (newRichTextRegex != null 

src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java

@@ -638,11 +638,6 @@ public Map<String, InputStream> getAttachments() {
         return this.jobUtil.getAttachments();
     }
 
-    @Override
-    protected int getMaxBytesInBatch() {
-        return this.getConfig().isBulkV2APIEnabled() ? AppConfig.MAX_BULKV2_API_IMPORT_JOB_BYTES : AppConfig.MAX_BULK_API_IMPORT_BATCH_BYTES;
-    }
-
     @Override
     protected int getBytesInBean(DynaBean dynaBean) {
         final ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -664,4 +659,9 @@ protected int getBytesInBean(DynaBean dynaBean) {
         }
         return beanBytes;
     }
+
+    @Override
+    protected int getMaxBytesInBatch() {
+        return controller.getAppConfig().getMaxBytesInBulkBatch();
+    }
 }

src/main/java/com/salesforce/dataloader/config/AppConfig.java

@@ -146,7 +146,8 @@ public class AppConfig {
     // Bulk v1 and v2 limits specified at https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_bulkapi.htm
     public static final int MAX_BULK_API_IMPORT_BATCH_BYTES = 10000000;
     public static final int MAX_NUM_ROWS_BULK_API_IMPORT_BATCH = 10000;
-    public static final int MAX_BULKV2_API_IMPORT_JOB_BYTES = 150000000;
+    public static final int DEFAULT_MAX_BULKV2_API_IMPORT_JOB_BYTES = 150000000;
+    private static int currentMaxBulkv2APIImportJobBytes = DEFAULT_MAX_BULKV2_API_IMPORT_JOB_BYTES;
     public static final int MAX_NUM_ROWS_BULKV2_API_IMPORT_JOB = 150000000;
     public static final int DEFAULT_NUM_ROWS_BULK_API_IMPORT_BATCH = 2000;
 
@@ -1675,7 +1676,7 @@ public boolean isBatchMode() {
         return (AppUtil.getAppRunMode() == AppUtil.APP_RUN_MODE.BATCH);
     }
 
-    public int getMaxRowsInImportBatch() {
+    public int getCurrentSettingForMaxRowsInImportBatch() {
         boolean bulkApi = isBulkAPIEnabled();
         boolean bulkV2Api = this.isBulkV2APIEnabled();
 
@@ -1689,17 +1690,17 @@ public int getMaxRowsInImportBatch() {
         } catch (ParameterLoadException e) {
         }
         int maxBatchSize = bulkApi ? MAX_NUM_ROWS_BULK_API_IMPORT_BATCH : MAX_NUM_ROWS_SOAP_API_IMPORT_BATCH;
-        return bs > maxBatchSize ? maxBatchSize : bs > 0 ? bs : getDefaultImportBatchSize(bulkApi, bulkV2Api);
+        return bs > maxBatchSize ? maxBatchSize : bs > 0 ? bs : getDefaultNumRowsImportBatch(bulkApi, bulkV2Api);
     }
 
-    public int getDefaultImportBatchSize(boolean bulkApi, boolean bulkV2Api) {
+    public int getDefaultNumRowsImportBatch(boolean bulkApi, boolean bulkV2Api) {
         if (bulkV2Api) {
             return MAX_NUM_ROWS_BULKV2_API_IMPORT_JOB;
         }
         return bulkApi ? DEFAULT_NUM_ROWS_BULK_API_IMPORT_BATCH : DEFAULT_NUM_ROWS_LOAD_BATCH;
     }
 
-    public int getMaxImportBatchSize(boolean bulkApi, boolean bulkV2Api) {
+    public int getMaxPossibleNumRowsImportBatchForAPIType(boolean bulkApi, boolean bulkV2Api) {
         if (bulkV2Api) {
             return MAX_NUM_ROWS_BULKV2_API_IMPORT_JOB;
         }
@@ -2003,4 +2004,12 @@ public static AppConfig getCurrentConfig() {
     public static interface ConfigListener {
         void configValueChanged(String key, String oldValue, String newValue);
     }
+
+    public static void setBulkV2JobMaxBytes(int maxBytes) {
+        currentMaxBulkv2APIImportJobBytes = maxBytes;
+    }
+
+    public int getMaxBytesInBulkBatch() {
+        return isBulkV2APIEnabled() ? AppConfig.currentMaxBulkv2APIImportJobBytes : AppConfig.MAX_BULK_API_IMPORT_BATCH_BYTES;
+    }
 }

src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java

@@ -345,11 +345,11 @@ public void widgetSelected(SelectionEvent e) {
                 setAllApiOptions();
 
                 // update batch size when this setting changes
-                int newDefaultBatchSize = getController().getAppConfig().getDefaultImportBatchSize(false, false);
+                int newDefaultBatchSize = getController().getAppConfig().getDefaultNumRowsImportBatch(false, false);
                 logger.debug("Setting batch size to " + newDefaultBatchSize);
                 textImportBatchSize.setText(String.valueOf(newDefaultBatchSize));
                 String[] args = {getImportBatchLimitsURL(), 
-                        Integer.toString(appConfig.getMaxImportBatchSize(useBulkAPI || useBulkV2API, useBulkV2API))};
+                        Integer.toString(appConfig.getMaxPossibleNumRowsImportBatchForAPIType(useBulkAPI || useBulkV2API, useBulkV2API))};
                 labelImportBatchSize.setText(
                         Labels.getFormattedString(AdvancedSettingsDialog.class.getSimpleName() + ".uiLabel." + AppConfig.PROP_IMPORT_BATCH_SIZE, args));
                 labelImportBatchSize.redraw();
@@ -377,11 +377,11 @@ public void widgetSelected(SelectionEvent e) {
                 setAllApiOptions();
 
                 // update batch size when this setting changes
-                int newDefaultBatchSize = getController().getAppConfig().getDefaultImportBatchSize(true, false);
+                int newDefaultBatchSize = getController().getAppConfig().getDefaultNumRowsImportBatch(true, false);
                 logger.debug("Setting batch size to " + newDefaultBatchSize);
                 textImportBatchSize.setText(String.valueOf(newDefaultBatchSize));
                 String[] args = {getImportBatchLimitsURL(), 
-                        Integer.toString(appConfig.getMaxImportBatchSize(useBulkAPI || useBulkV2API, useBulkV2API))};
+                        Integer.toString(appConfig.getMaxPossibleNumRowsImportBatchForAPIType(useBulkAPI || useBulkV2API, useBulkV2API))};
                 labelImportBatchSize.setText(
                         Labels.getFormattedString(AdvancedSettingsDialog.class.getSimpleName() + ".uiLabel." + AppConfig.PROP_IMPORT_BATCH_SIZE, args));
                 labelImportBatchSize.redraw();
@@ -410,11 +410,11 @@ public void widgetSelected(SelectionEvent e) {
                 setAllApiOptions();
 
                 // get default batch size for Bulk v2 and set it
-                int newDefaultBatchSize = getController().getAppConfig().getDefaultImportBatchSize(true, true);
+                int newDefaultBatchSize = getController().getAppConfig().getDefaultNumRowsImportBatch(true, true);
                 logger.debug("Setting batch size to " + newDefaultBatchSize);
                 textImportBatchSize.setText(String.valueOf(newDefaultBatchSize));
                 String[] args = {getImportBatchLimitsURL(), 
-                        Integer.toString(appConfig.getMaxImportBatchSize(useBulkAPI || useBulkV2API, useBulkV2API))};
+                        Integer.toString(appConfig.getMaxPossibleNumRowsImportBatchForAPIType(useBulkAPI || useBulkV2API, useBulkV2API))};
                 labelImportBatchSize.setText(
                         Labels.getFormattedString(AdvancedSettingsDialog.class.getSimpleName() + ".uiLabel." + AppConfig.PROP_IMPORT_BATCH_SIZE, args));
                 labelImportBatchSize.redraw();
@@ -537,10 +537,10 @@ public void verifyText(VerifyEvent event) {
         this.importBatchSizeComposite.setLayout(layout);
 
         String[] args = {getImportBatchLimitsURL(), 
-                Integer.toString(appConfig.getMaxImportBatchSize(useBulkAPI || useBulkV2API, useBulkV2API))};
+                Integer.toString(appConfig.getMaxPossibleNumRowsImportBatchForAPIType(useBulkAPI || useBulkV2API, useBulkV2API))};
         labelImportBatchSize = createLink(importBatchSizeComposite, null, args, AppConfig.PROP_IMPORT_BATCH_SIZE);
         textImportBatchSize = new Text(importBatchSizeComposite, SWT.BORDER);
-        textImportBatchSize.setText(Integer.toString(appConfig.getMaxRowsInImportBatch()));
+        textImportBatchSize.setText(Integer.toString(appConfig.getCurrentSettingForMaxRowsInImportBatch()));
         textImportBatchSize.setEnabled(!useBulkV2API);
         textImportBatchSize.addVerifyListener(new VerifyListener() {
             @Override

src/main/java/com/salesforce/dataloader/ui/DataSelectionDialog.java

@@ -167,7 +167,7 @@ protected void processingWithBusyIndicator(Shell shell) {
                 + "\n\n"
                 + Labels.getString("LoadPage.importBatchSize")
                 + " "
-                + getController().getAppConfig().getMaxRowsInImportBatch()
+                + getController().getAppConfig().getCurrentSettingForMaxRowsInImportBatch()
                 + "\n"
                 + Labels.getString("AdvancedSettingsDialog.uiLabel." + AppConfig.PROP_LOAD_ROW_TO_START_AT)
                 + " "

src/main/java/com/salesforce/dataloader/ui/LoadPage.java

@@ -53,7 +53,7 @@ public LoadPage(String name, Controller controller) {
     protected String getConfigInfo() {
         return Labels.getString("LoadPage.importBatchSize")
         + " "
-        + controller.getAppConfig().getMaxRowsInImportBatch()
+        + controller.getAppConfig().getCurrentSettingForMaxRowsInImportBatch()
         + "    "
         + Labels.getString("AdvancedSettingsDialog.uiLabel." + AppConfig.PROP_LOAD_ROW_TO_START_AT)
         + " "

src/test/java/com/salesforce/dataloader/process/CsvProcessTest.java

@@ -222,8 +222,29 @@ public void testUpsertAccountCsv() throws Exception {
         }
         // manually inserts 50 accounts, then upserts 100 accounts (50 inserts and 50 updates)
         runUpsertProcess(configMap, 50, 50);
+        if (isSettingEnabled(configMap, AppConfig.PROP_BULKV2_API_ENABLED)) {
+            // test ability to run multiple Bulk v2 jobs
+            AppConfig.setBulkV2JobMaxBytes(1500);
+            runUpsertProcess(configMap, 0, 100);
+            AppConfig.setBulkV2JobMaxBytes(AppConfig.DEFAULT_MAX_BULKV2_API_IMPORT_JOB_BYTES);
+       }
     }
-
+
+    /**
+     * test ability to run multiple Bulk v2 jobs
+     */
+    @Test
+    public void testBulkV2MultipleJobsCsv() throws Exception {
+        Map<String, String> configMap = getUpdateTestConfig(true, DEFAULT_ACCOUNT_EXT_ID_FIELD, 50);
+        if (!isSettingEnabled(configMap, AppConfig.PROP_BULKV2_API_ENABLED)) {
+            return;
+        }
+        AppConfig.setBulkV2JobMaxBytes(1500);
+        // creates multiple bulk v2 jobs while creating 50 accounts, then upserting 100 accounts (50 inserts and 50 updates)
+        runUpsertProcess(configMap, 50, 50);
+        AppConfig.setBulkV2JobMaxBytes(AppConfig.DEFAULT_MAX_BULKV2_API_IMPORT_JOB_BYTES);
+    }
+
     /**
      * Verify that when the constants are placed into the .sdl mapping file,
      * they are treated correctly. It tests that constants mapped on 1 field and

src/test/resources/testfiles/data/bulkV2MultipleJobsCsvMap.sdl

@@ -0,0 +1,9 @@
+#Mapping values
+#Tue Jun 20 18:59:17 PDT 2006
+ACCOUNTNUMBER__C=AccountNumber__c
+NAME=Name
+WEBSITE=Website
+TYPE=Type
+ANNUALREVENUE=AnnualRevenue
+PHONE=Phone
+ORACLE_ID__C=Oracle_Id__c