add default max batch size and batchSize parameter for SF updates

.github/codeowners

@@ -0,0 +1,2 @@
+* @loanpal-engineering/business-intelligence-bi
+.github/codeowners @loanpal-engineering/security @loanpal-engineering/DevOps

.gitignore

@@ -10,3 +10,6 @@ target/
 project/target
 dependency-reduced-pom.xml
 /bin/
+.DS_Store
+.databricks
+.vscode/

README.md

@@ -62,6 +62,8 @@ $ bin/spark-shell --packages com.springml:spark-salesforce_2.11:1.1.3
 * `timeout`: (Optional) The maximum time spent polling for the completion of bulk query job. This option can only be used when `bulk` is `true`.
 * `externalIdFieldName`: (Optional) The name of the field used as the external ID for Salesforce Object. This value is only used when doing an update or upsert. Default "Id".
 * `queryAll`: (Optional) Toggle to retrieve deleted and archived records for SOQL queries. Default value is `false`.
+### Options only supported for fetching Salesforce Objects.
+* `batchSize`: (Optional) maximum number of records per batch when performing updates. Defaults to 5000 (note that batches greater than 10000 will result in a error)
 
 
 ### Scala API

pom.xml

@@ -0,0 +1,202 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.goodleap</groupId>
+    <artifactId>spark-salesforce</artifactId>
+    <packaging>jar</packaging>
+    <description>spark-salesforce</description>
+    <version>1.1.6</version>
+    <name>spark-salesforce</name>
+    <organization>
+        <name>com.springml</name>
+    </organization>
+    <url>https://github.com/springml/spark-salesforce</url>
+    <licenses>
+        <license>
+            <name>Apache License, Verision 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+    <scm>
+        <connection>scm:git:github.com/springml/spark-salesforce</connection>
+        <developerConnection>scm:git:git@github.com:springml/spark-salesforce</developerConnection>
+        <url>github.com/springml/spark-salesforce</url>
+    </scm>
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <encoding>UTF-8</encoding>
+        <spark.version>3.3.0</spark.version>
+        <scala.version>2.12.11</scala.version>
+        <scala.compat.version>2.12</scala.compat.version>
+    </properties>
+    <developers>
+        <developer>
+            <id>springml</id>
+            <name>Springml</name>
+            <url>http://www.springml.com</url>
+        </developer>
+    </developers>
+    <dependencies>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.compat.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <!-- -->
+        <dependency>
+            <groupId>com.force.api</groupId>
+            <artifactId>force-wsc</artifactId>
+            <version>53.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.force.api</groupId>
+            <artifactId>force-partner-api</artifactId>
+            <version>53.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.springml</groupId>
+            <artifactId>salesforce-wave-api</artifactId>
+            <version>1.0.8-loanpal</version>
+        </dependency>
+        <!--        <dependency>-->
+        <!--            <groupId>com.github.loanpal-engineering</groupId>-->
+        <!--            <artifactId>salesforce-wave-api</artifactId>-->
+        <!--            <version>1d662ac</version>-->
+        <!--&lt;!&ndash;            <exclusions>&ndash;&gt;-->
+        <!--&lt;!&ndash;                <exclusion>&ndash;&gt;-->
+        <!--&lt;!&ndash;                    <groupId>org.codehaus.woodstox</groupId>&ndash;&gt;-->
+        <!--&lt;!&ndash;                    <artifactId>stax2-api</artifactId>&ndash;&gt;-->
+        <!--&lt;!&ndash;                </exclusion>&ndash;&gt;-->
+        <!--&lt;!&ndash;            </exclusions>&ndash;&gt;-->
+        <!--        </dependency>-->
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>2.0.31-beta</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.12</artifactId>
+            <version>3.0.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>endolabs.salesforce</groupId>
+            <artifactId>bulkv2</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>3.14.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>logging-interceptor</artifactId>
+            <version>3.14.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.frejo</groupId>
+            <artifactId>force-rest-api</artifactId>
+            <version>0.0.42</version>
+        </dependency>
+        <dependency>
+            <groupId>org.codehaus.woodstox</groupId>
+            <artifactId>woodstox-core-asl</artifactId>
+            <version>4.4.1</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.codehaus.woodstox</groupId>
+                    <artifactId>stax2-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <!--        <dependency>-->
+        <!--            <groupId>org.codehaus.woodstox</groupId>-->
+        <!--            <artifactId>stax2-api</artifactId>-->
+        <!--            <version>4.2.1</version>-->
+        <!--        </dependency>-->
+    </dependencies>
+    <build>
+        <sourceDirectory>src/main/scala</sourceDirectory>
+        <plugins>
+            <plugin>
+                <!-- see http://davidb.github.com/scala-maven-plugin -->
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>3.4.6</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                        <configuration>
+                            <jvmArgs>
+                                <jvmArg>-Xss16m</jvmArg>
+                                <jvmArg>-Xms1028m</jvmArg>
+                                <jvmArg>-Xmx4096m</jvmArg>
+                            </jvmArgs>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.4.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.fasterxml.jackson.dataformat</pattern>
+                                    <shadedPattern>com.shaded.fasterxml.jackson.dataformat</shadedPattern>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                    <finalName>uber-${project.artifactId}-${project.version}</finalName>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

src/main/scala/com/springml/spark/salesforce/BulkRelation.scala

@@ -34,7 +34,9 @@ case class BulkRelation(
     userSchema: StructType,
     sqlContext: SQLContext,
     inferSchema: Boolean,
-    timeout: Long) extends BaseRelation with TableScan {
+    timeout: Long,
+    maxCharsPerColumn: Int,
+    maxColumns: Int) extends BaseRelation with TableScan {
 
   import sqlContext.sparkSession.implicits._
 
@@ -71,12 +73,15 @@ case class BulkRelation(
 
         // Use Csv parser to split CSV by rows to cover edge cases (ex. escaped characters, new line within string, etc)
         def splitCsvByRows(csvString: String): Seq[String] = {
+          if (csvString == "Records not found for this query") Seq.empty
           // The CsvParser interface only interacts with IO, so StringReader and StringWriter
           val inputReader = new StringReader(csvString)
 
           val parserSettings = new CsvParserSettings()
           parserSettings.setLineSeparatorDetectionEnabled(true)
           parserSettings.getFormat.setNormalizedNewline(' ')
+          parserSettings.setMaxCharsPerColumn(maxCharsPerColumn)
+          parserSettings.setMaxColumns(maxColumns)
 
           val readerParser = new CsvParser(parserSettings)
           val parsedInput = readerParser.parseAll(inputReader).asScala
@@ -90,7 +95,7 @@ case class BulkRelation(
           val writer = new CsvWriter(outputWriter, writerSettings)
           parsedInput.foreach { writer.writeRow(_) }
 
-          outputWriter.toString.lines.toList
+          outputWriter.toString.split("\n").toList
         }
 
         splitCsvByRows(result)
@@ -109,6 +114,7 @@ case class BulkRelation(
         .option("quote", "\"")
         .option("escape", "\"")
         .option("multiLine", true)
+        .option("maxColumns", maxColumns)
         .csv(csvData)
     } else {
       bulkAPI.closeJob(jobId)