Merge pull request #345 from splunk/timestamp-parsing

Update README.md

Author: foram-splunk

README.md

@@ -135,6 +135,9 @@ Use the below schema to configure Splunk Connect for Kafka
    "splunk.hec.ssl.trust.store.password": "<Java KeyStore password>"
    "kerberos.user.principal": "<The Kerberos user principal the connector may use to authenticate with Kerberos>",
    "kerberos.keytab.path": "<The path to the keytab file to use for authentication with Kerberos>"
+   "enable.timestamp.extraction": "<true|false>",
+   "timestamp.regex": "<regex for timestamp extraction>",
+   "timestamp.format": "<time-format for timestamp extraction>"
   }
 }
 ```
@@ -222,6 +225,13 @@ Use the below schema to configure Splunk Connect for Kafka
 | `key.converter.schema.registry.url` |  Schema Registry URL. | `""` |
 | `key.converter.schemas.enable` | For using protobuf format ,set the value of this field to `true` | `false` |
 
+### Timestamp extraction Parameters
+| Name              | Description                | Default Value  |
+|--------           |----------------------------|-----------------------|
+| `enable.timestamp.extraction` |  To enable timestamp extraction ,set the value of this field to `true`. <br/> **NOTE:** <br/> Applicable only if `splunk.hec.raw` is `false` | `false` |
+| `timestamp.regex` |  Regex for timestamp extraction. <br/> **NOTE:** <br/> Regex must have name captured group `"time"` For eg.: `\\\"time\\\":\\s*\\\"(?<time>.*?)\"` | `""` |
+| `timestamp.format` |  Time-format for timestamp extraction .<br/>For eg.: <br/>If timestamp is `1555209605000` , set `timestamp.format` to `"epoch"` format .<br/> If timestamp is `Jun 13 2010 23:11:52.454 UTC` , set `timestamp.format` to `"MMM dd yyyy HH:mm:ss.SSS zzz"` | `""` |
+
 ## Load balancing
 
 See [Splunk Docs](https://docs.splunk.com/Documentation/KafkaConnect/latest/User/LoadBalancing) for considerations when using load balancing in your deployment.

src/main/java/com/splunk/kafka/connect/SplunkSinkConnectorConfig.java

@@ -91,7 +91,7 @@ public final class SplunkSinkConnectorConfig extends AbstractConfig {
 
      // Input the Regex String and timestamp format
      static final String ENABLE_TIMESTAMP_EXTRACTION_CONF = "enable.timestamp.extraction";
-     static final String REGEX_CONF = "regex";
+     static final String REGEX_CONF = "timestamp.regex";
      static final String TIMESTAMP_FORMAT_CONF = "timestamp.format";
 
     // Kafka configuration description strings

test/lib/connect_params.py

@@ -206,13 +206,13 @@
      "topics": "date_format",
      "splunk_hec_raw": False,
      "enable_timestamp_extraction" : "true",
-     "regex": r"\\\"time\\\":\\s*\\\"(?<time>.*?)\"",
+     "timestamp_regex": r"\\\"time\\\":\\s*\\\"(?<time>.*?)\"",
      "timestamp_format": "MMM dd yyyy HH:mm:ss.SSS zzz"},
     {"name": "test_extracted_timestamp_epochformat",
      "splunk_sourcetypes": "epoch_format",
      "topics": "epoch_format",
      "splunk_hec_raw": False,
      "enable_timestamp_extraction" : "true",
-     "regex": r"\\\"time\\\":\\s*\\\"(?<time>.*?)\"",
+     "timestamp_regex": r"\\\"time\\\":\\s*\\\"(?<time>.*?)\"",
      "timestamp_format": "epoch"}
 ]

test/lib/connector.template

@@ -49,7 +49,7 @@
             "value.converter.schema.registry.url": "{{value_converter_schema_registry_url}}",
             "value.converter.schemas.enable": "{{value_converter_schemas_enable}}",
 			"enable.timestamp.extraction": "{{enable_timestamp_extraction}}",
-			"regex": "{{regex}}",
+			"timestamp.regex": "{{timestamp_regex}}",
 			"timestamp.format": "{{timestamp_format}}"
 		}
 }