diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index ac4a5ec4d4721..5f5408d7fa189 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -301,7 +301,15 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas String first = commands[0].trim(); // If true, we're using *:index, otherwise we're using *:index,index boolean onlyRemotes = canUseRemoteIndicesOnly() && randomBoolean(); - String[] commandParts = first.split("\\s+", 2); + + // Split "SET a=b; FROM x" into "SET a=b" and "FROM x" + int lastSetDelimiterPosition = first.lastIndexOf(';'); + String setStatements = lastSetDelimiterPosition == -1 ? "" : first.substring(0, lastSetDelimiterPosition + 1); + String afterSetStatements = lastSetDelimiterPosition == -1 ? first : first.substring(lastSetDelimiterPosition + 1); + + // Split "FROM a, b, c" into "FROM" and "a, b, c" + String[] commandParts = afterSetStatements.trim().split("\\s+", 2); + String command = commandParts[0].trim(); if (command.equalsIgnoreCase("from") || command.equalsIgnoreCase("ts")) { String[] indexMetadataParts = commandParts[1].split("(?i)\\bmetadata\\b", 2); @@ -323,7 +331,7 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas + remoteIndices + " " + (indexMetadataParts.length == 1 ? "" : "metadata " + indexMetadataParts[1]); - testCase.query = newFirstCommand + query.substring(first.length()); + testCase.query = setStatements + newFirstCommand + query.substring(first.length()); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java index ba0d11059a69b..0af1ac662fa69 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java @@ -13,9 +13,6 @@ import java.util.function.Function; import java.util.regex.Pattern; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - public final class CsvSpecReader { private CsvSpecReader() {} @@ -25,9 +22,6 @@ public static SpecReader.Parser specParser() { } public static class CsvSpecParser implements SpecReader.Parser { - private static final String SCHEMA_PREFIX = "schema::"; - - private final StringBuilder earlySchema = new StringBuilder(); private final StringBuilder query = new StringBuilder(); private final StringBuilder data = new StringBuilder(); private final List requiredCapabilities = new ArrayList<>(); @@ -39,21 +33,22 @@ private CsvSpecParser() {} public Object parse(String line) { // read the query if (testCase == null) { - if (line.startsWith(SCHEMA_PREFIX)) { - assertThat("Early schema already declared " + earlySchema, earlySchema.length(), is(0)); - earlySchema.append(line.substring(SCHEMA_PREFIX.length()).trim()); - } else if (line.toLowerCase(Locale.ROOT).startsWith("required_capability:")) { + if (line.toLowerCase(Locale.ROOT).startsWith("required_capability:")) { requiredCapabilities.add(line.substring("required_capability:".length()).trim()); } else { - if (line.endsWith(";")) { + if (line.endsWith("\\;")) { + // SET statement with escaped ";" + var updatedLine = line.substring(0, line.length() - 2); + query.append(updatedLine); + query.append(";"); + query.append("\r\n"); + } else if (line.endsWith(";")) { // pick up the query testCase = new CsvTestCase(); query.append(line.substring(0, line.length() - 1).trim()); testCase.query = query.toString(); - testCase.earlySchema = earlySchema.toString(); testCase.requiredCapabilities = List.copyOf(requiredCapabilities); requiredCapabilities.clear(); - earlySchema.setLength(0); query.setLength(0); } // keep reading the query @@ -109,7 +104,6 @@ private static Pattern warningRegexToPattern(String regex) { public static class CsvTestCase { public String query; - public String earlySchema; public String expectedResults; private final List expectedWarnings = new ArrayList<>(); private final List expectedWarningsRegexString = new ArrayList<>(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 6287ad9703fe1..68810e0686813 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -84,7 +84,6 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.util.DateUtils; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; @@ -107,7 +106,9 @@ import org.elasticsearch.xpack.esql.inference.InferenceService; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.parser.QueryParam; +import org.elasticsearch.xpack.esql.plan.EsqlStatement; import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.QuerySettings; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Explain; @@ -563,9 +564,9 @@ private static ThreadPool createMockThreadPool() { private EsqlTestUtils() {} - public static Configuration configuration(QueryPragmas pragmas, String query) { + public static Configuration configuration(QueryPragmas pragmas, String query, EsqlStatement statement) { return new Configuration( - DateUtils.UTC, + statement.setting(QuerySettings.TIME_ZONE), Locale.US, null, null, @@ -582,12 +583,16 @@ public static Configuration configuration(QueryPragmas pragmas, String query) { ); } + public static Configuration configuration(QueryPragmas pragmas, String query) { + return configuration(pragmas, query, new EsqlStatement(null, List.of())); + } + public static Configuration configuration(QueryPragmas pragmas) { return configuration(pragmas, StringUtils.EMPTY); } public static Configuration configuration(String query) { - return configuration(new QueryPragmas(Settings.EMPTY), query); + return configuration(QueryPragmas.EMPTY, query); } public static AnalyzerSettings queryClusterSettings() { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 8beef707706ca..c3bf0d9d0ebbf 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -55,7 +55,6 @@ emp_no:integer | x:keyword | y:keyword 10061 | 1985-09-17T00:00:00.000Z | 1985-09-17 ; - compareToString from employees | where hire_date < "1985-03-01T00:00:00Z" | keep emp_no, hire_date; ignoreOrder:true @@ -1085,6 +1084,33 @@ date:date | year:long 2022-05-06T00:00:00.000Z | 2022 ; +dateExtractSetTimezoneFrom +required_capability: global_timezone_parameter + +set time_zone="+07:00"\; +from employees +| sort hire_date +| eval hour = date_extract("hour_of_day", hire_date) +| keep emp_no, hire_date, hour +| limit 2; + +emp_no:integer | hire_date:date | hour:long +10009 | 1985-02-18T00:00:00.000Z | 7 +10048 | 1985-02-24T00:00:00.000Z | 7 +; + +dateExtractSetTimezoneRow +required_capability: global_timezone_parameter + +set time_zone="+07:00"\; +ROW hire_date = "2020-02-28T23:00:00.000Z"::date +| EVAL hour = date_extract("hour_of_day", hire_date) +| KEEP hour; + +hour:long +6 +; + docsDateExtractBusinessHours // tag::docsDateExtractBusinessHours[] FROM sample_data @@ -1971,6 +1997,30 @@ emp_no:integer | birth_date:date | hire_date:date 10040 | null | 1993-02-14T00:00:00.000Z | null ; +dayNameSetTimezoneRow +required_capability: global_timezone_parameter + +set time_zone="-02:00"\; +row dt = to_datetime("1953-09-02T00:00:00.000Z") +| eval weekday = day_name(dt); + +dt:date | weekday:keyword +1953-09-02T00:00:00.000Z | Tuesday +; + +dayNameSetTimezoneFrom +required_capability: global_timezone_parameter + +set time_zone="-02:00"\; +from employees +| sort emp_no +| keep emp_no, hire_date +| eval day = day_name(hire_date) +| limit 1; + +emp_no:integer | hire_date:date | day:keyword +10001 | 1986-06-26T00:00:00.000Z | Wednesday +; monthNameRowTest required_capability:fn_month_name @@ -2068,3 +2118,28 @@ from employees emp_no:integer | birth_date:date | hire_date:date | monthName:keyword 10040 | null | 1993-02-14T00:00:00.000Z | null ; + +monthNameSetTimezoneRow +required_capability: global_timezone_parameter + +set time_zone="Europe/Paris"\; +row dt = to_datetime("1996-01-31T23:00:00.000Z") +| eval monthName = MONTH_NAME(dt); + +dt:date | monthName:keyword +1996-01-31T23:00:00.000Z | February +; + +monthNameSetTimezoneFrom +required_capability: global_timezone_parameter + +set time_zone="-10:00"\; +from employees +| WHERE emp_no == 10004 +| keep emp_no, hire_date +| eval monthName = month_name(hire_date) +| limit 1; + +emp_no:integer | hire_date:date | monthName:keyword +10004 | 1986-12-01T00:00:00.000Z | November +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/set.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/set.csv-spec new file mode 100644 index 0000000000000..24702a83ce79f --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/set.csv-spec @@ -0,0 +1,40 @@ +set +required_capability: global_timezone_parameter + +set time_zone="+02:00"\; +from employees +| sort emp_no +| keep emp_no, hire_date +| eval hour = date_extract("hour_of_day", hire_date) +| limit 1; + +emp_no:integer | hire_date:date | hour:long +10001 | 1986-06-26T00:00:00.000Z | 2 +; + +set with foldable +required_capability: global_timezone_parameter + +set time_zone="+02:00"\; +ROW date = "1986-06-26T00:00:00.000Z"::date +| eval hour = date_extract("hour_of_day", date) +; + +date:date | hour:long +1986-06-26T00:00:00.000Z | 2 +; + +last set prevails +required_capability: global_timezone_parameter + +set time_zone="+02:00"\; +set time_zone="+05:00"\; +from employees +| sort emp_no +| keep emp_no, hire_date +| eval hour = date_extract("hour_of_day", hire_date) +| limit 1; + +emp_no:integer | hire_date:date | hour:long +10001 | 1986-06-26T00:00:00.000Z | 5 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 803e7c8e1dff7..698aaac21218f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1548,6 +1548,11 @@ public enum Cap { */ FIX_FILTER_ORDINALS, + /** + * "time_zone" parameter in request body and in {@code SET "time_zone"="x"} + */ + GLOBAL_TIMEZONE_PARAMETER(Build.current().isSnapshot()), + /** * Optional options argument for DATE_PARSE */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 1c844d3c15f6c..0c0a6a18c9d11 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -115,6 +115,11 @@ public LogicalPlan createStatement(String query, QueryParams params, PlanTelemet return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan); } + // testing utility + public EsqlStatement createQuery(String query) { + return createQuery(query, new QueryParams()); + } + // testing utility public EsqlStatement createQuery(String query, QueryParams params) { return createQuery(query, params, new PlanTelemetry(new EsqlFunctionRegistry())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java index 7e2dea975f326..9d248ffc19abe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java @@ -202,7 +202,7 @@ public long absoluteStartedTimeInMillis() { /** * @return Start time of the ESQL query in nanos */ - public long getQueryStartTimeNanos() { + public long queryStartTimeNanos() { return queryStartTimeNanos; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index d90b59eaf38ff..8685df23d5089 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -80,6 +80,7 @@ import org.elasticsearch.xpack.esql.optimizer.LogicalPreOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.TestLocalPhysicalPlanOptimizer; import org.elasticsearch.xpack.esql.parser.EsqlParser; +import org.elasticsearch.xpack.esql.plan.EsqlStatement; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -191,9 +192,13 @@ public class CsvTests extends ESTestCase { private final CsvSpecReader.CsvTestCase testCase; private final String instructions; - private final Configuration configuration = EsqlTestUtils.configuration( - new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()) - ); + /** + * The configuration to be used in the tests. + *

+ * Initialized in {@link #executePlan}. + *

+ */ + private Configuration configuration; private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); private final EsqlParser parser = new EsqlParser(); private final Mapper mapper = new Mapper(); @@ -551,6 +556,7 @@ private static EnrichPolicy loadEnrichPolicyMapping(String policyFileName) { private LogicalPlan analyzedPlan( LogicalPlan parsed, + Configuration configuration, Map datasets, TransportVersion minimumVersion ) { @@ -638,11 +644,16 @@ private static TestPhysicalOperationProviders testOperationProviders( } private ActualResults executePlan(BigArrays bigArrays) throws Exception { - LogicalPlan parsed = parser.createStatement(testCase.query); - var testDatasets = testDatasets(parsed); + EsqlStatement statement = parser.createQuery(testCase.query); + this.configuration = EsqlTestUtils.configuration( + new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()), + testCase.query, + statement + ); + var testDatasets = testDatasets(statement.plan()); // Specifically use the newest transport version; the csv tests correspond to a single node cluster on the current version. TransportVersion minimumVersion = TransportVersion.current(); - LogicalPlan analyzed = analyzedPlan(parsed, testDatasets, minimumVersion); + LogicalPlan analyzed = analyzedPlan(statement.plan(), configuration, testDatasets, minimumVersion); FoldContext foldCtx = FoldContext.small(); EsqlSession session = new EsqlSession( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index f848903688195..4d62c04528c55 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -116,6 +116,8 @@ protected static List withNoRowsExpectingNull(List td.isMultiRow() ? td.withData(List.of()) : td).toList(); return new TestCaseSupplier.TestCase( + testCase.getSource(), + testCase.getConfiguration(), newData, testCase.evaluatorToString(), testCase.expectedType(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 3a5d49c4cc188..52c541cf7d384 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -180,6 +180,8 @@ protected static List anyNullIsNull( }).toList(); TestCaseSupplier.TypedData nulledData = oc.getData().get(finalNullPosition); return new TestCaseSupplier.TestCase( + oc.getSource(), + oc.getConfiguration(), data, evaluatorToString.evaluatorToString(finalNullPosition, nulledData, oc.evaluatorToString()), expectedType.expectedType(finalNullPosition, nulledData.type(), oc), @@ -212,6 +214,8 @@ protected static List anyNullIsNull( ) .toList(); return new TestCaseSupplier.TestCase( + oc.getSource(), + oc.getConfiguration(), data, equalTo("LiteralsEvaluator[lit=null]"), expectedType.expectedType(finalNullPosition, DataType.NULL, oc), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 3d75db67e7604..04c6811619ca0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -23,6 +23,7 @@ import org.elasticsearch.logging.Logger; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.ConfigurationTestUtils; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -42,8 +43,10 @@ import java.time.Period; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.BiFunction; import java.util.function.BinaryOperator; import java.util.function.DoubleFunction; @@ -1570,6 +1573,15 @@ public static String castToDoubleEvaluator(String original, DataType current) { throw new UnsupportedOperationException(); } + public static List mapTestCases( + Collection suppliers, + Function mapper + ) { + return suppliers.stream() + .map(supplier -> new TestCaseSupplier(supplier.name(), supplier.types(), () -> mapper.apply(supplier.get()))) + .toList(); + } + public static final class TestCase { /** * The {@link Source} this test case should be run with @@ -1655,6 +1667,8 @@ public static TestCase typeError(List data, String expectedTypeError) Object extra ) { this( + TEST_SOURCE, + ConfigurationTestUtils.randomConfiguration(TEST_SOURCE.text(), Map.of()), data, evaluatorToString, expectedType, @@ -1670,6 +1684,8 @@ public static TestCase typeError(List data, String expectedTypeError) } TestCase( + Source source, + Configuration configuration, List data, Matcher evaluatorToString, DataType expectedType, @@ -1682,8 +1698,8 @@ public static TestCase typeError(List data, String expectedTypeError) Object extra, boolean canBuildEvaluator ) { - this.source = TEST_SOURCE; - this.configuration = TEST_CONFIGURATION; + this.source = source; + this.configuration = configuration; this.data = data; this.evaluatorToString = evaluatorToString; this.expectedType = expectedType == null ? null : expectedType.noText(); @@ -1779,11 +1795,50 @@ public Object extra() { return extra; } + /** + * Build a new {@link TestCase} with the {@link #TEST_CONFIGURATION}. + *

+ * The source is also set to match the configuration + *

+ * + * @deprecated Use a custom configuration instead, and test the results. + */ + @Deprecated + public TestCase withStaticConfiguration() { + return withConfiguration(TEST_SOURCE, TEST_CONFIGURATION); + } + + /** + * Build a new {@link TestCase} with new {@link #configuration}. + *

+ * As the configuration query should match the source, the source is also updated here. + *

+ */ + public TestCase withConfiguration(Source source, Configuration configuration) { + return new TestCase( + source, + configuration, + data, + evaluatorToString, + expectedType, + matcher, + expectedWarnings, + expectedBuildEvaluatorWarnings, + expectedTypeError, + foldingExceptionClass, + foldingExceptionMessage, + extra, + canBuildEvaluator + ); + } + /** * Build a new {@link TestCase} with new {@link #data}. */ public TestCase withData(List data) { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, @@ -1803,6 +1858,8 @@ public TestCase withData(List data) { */ public TestCase withExtra(Object extra) { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, @@ -1819,6 +1876,8 @@ public TestCase withExtra(Object extra) { public TestCase withWarning(String warning) { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, @@ -1839,6 +1898,8 @@ public TestCase withWarning(String warning) { */ public TestCase withBuildEvaluatorWarning(String warning) { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, @@ -1864,6 +1925,8 @@ private String[] addWarning(String[] warnings, String warning) { public TestCase withFoldingException(Class clazz, String message) { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, @@ -1886,6 +1949,8 @@ public TestCase withFoldingException(Class clazz, String me */ public TestCase withoutEvaluator() { return new TestCase( + source, + configuration, data, evaluatorToString, expectedType, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java index 239cd3ebfc219..116e8ee34e942 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/AbstractConfigurationFunctionTestCase.java @@ -7,18 +7,15 @@ package org.elasticsearch.xpack.esql.expression.function.scalar; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.esql.analysis.AnalyzerSettings; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; -import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.session.Configuration; import java.util.List; -import java.util.Map; +import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; +import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; public abstract class AbstractConfigurationFunctionTestCase extends AbstractScalarFunctionTestCase { @@ -32,34 +29,18 @@ protected Expression build(Source source, List args) { public void testSerializationWithConfiguration() { assumeTrue("can't serialize function", canSerialize()); - Configuration config = randomConfiguration(); + Configuration config = randomConfiguration(testCase.getSource().text(), randomTables()); Expression expr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), config); assertSerialization(expr, config); - Configuration differentConfig = randomValueOtherThan(config, AbstractConfigurationFunctionTestCase::randomConfiguration); + Configuration differentConfig = randomValueOtherThan( + config, + // The source must match the original (static) one, as function source serialization depends on it + () -> randomConfiguration(testCase.getSource().text(), randomTables()) + ); Expression differentExpr = buildWithConfiguration(testCase.getSource(), testCase.getDataAsFields(), differentConfig); assertNotEquals(expr, differentExpr); } - - private static Configuration randomConfiguration() { - // TODO: Randomize the query and maybe the pragmas. - return new Configuration( - randomZone(), - randomLocale(random()), - randomBoolean() ? null : randomAlphaOfLength(randomInt(64)), - randomBoolean() ? null : randomAlphaOfLength(randomInt(64)), - QueryPragmas.EMPTY, - AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY), - AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY), - StringUtils.EMPTY, - randomBoolean(), - Map.of(), - System.nanoTime(), - randomBoolean(), - AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY), - AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY) - ); - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index 12e829a0254ed..fb7ec0ddb3897 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", DataType.LONG, equalTo(2023L) - ) + ).withStaticConfiguration() ), new TestCaseSupplier( List.of(stringType, DataType.DATE_NANOS), @@ -68,7 +68,7 @@ public static Iterable parameters() { "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", DataType.LONG, equalTo(2023L) - ) + ).withStaticConfiguration() ), new TestCaseSupplier( List.of(stringType, DataType.DATE_NANOS), @@ -80,7 +80,7 @@ public static Iterable parameters() { "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", DataType.LONG, equalTo(123456L) - ) + ).withStaticConfiguration() ), new TestCaseSupplier( List.of(stringType, DataType.DATETIME), @@ -93,7 +93,10 @@ public static Iterable parameters() { "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", DataType.LONG, is(nullValue()) - ).withWarning("Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded.") + ).withStaticConfiguration() + .withWarning( + "Line 1:1: evaluation of [source] failed, treating result as null. Only first 20 failures recorded." + ) .withWarning( "Line 1:1: java.lang.IllegalArgumentException: " + "No enum constant java.time.temporal.ChronoField.NOT A UNIT" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index b8cc2a3ecacce..10ead11f09273 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -84,6 +84,7 @@ public static Iterable parameters() { (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatNanos(DateUtils.toLong((Instant) value))), List.of() ); + suppliers = TestCaseSupplier.mapTestCases(suppliers, testCase -> testCase.withStaticConfiguration()); return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DayNameTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DayNameTests.java index af86630487957..b50377660c8c5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DayNameTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DayNameTests.java @@ -62,7 +62,7 @@ public static Iterable parameters() { Matchers.startsWith("DayNameMillisEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(null) - ) + ).withStaticConfiguration() ) ); @@ -78,7 +78,7 @@ private static List generateTest(String dateTime, String expec Matchers.startsWith("DayNameMillisEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(new BytesRef(expectedWeekDay)) - ) + ).withStaticConfiguration() ), new TestCaseSupplier( List.of(DataType.DATE_NANOS), @@ -87,7 +87,7 @@ private static List generateTest(String dateTime, String expec Matchers.is("DayNameNanosEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(new BytesRef(expectedWeekDay)) - ) + ).withStaticConfiguration() ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/MonthNameTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/MonthNameTests.java index aefec9da055aa..9c4e6c2007b12 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/MonthNameTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/MonthNameTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { Matchers.startsWith("MonthNameMillisEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(null) - ) + ).withStaticConfiguration() ) ); @@ -88,7 +88,7 @@ private static List generateTest(String dateTime, String expec Matchers.startsWith("MonthNameMillisEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(new BytesRef(expectedMonthName)) - ) + ).withStaticConfiguration() ), new TestCaseSupplier( List.of(DataType.DATE_NANOS), @@ -97,7 +97,7 @@ private static List generateTest(String dateTime, String expec Matchers.is("MonthNameNanosEvaluator[val=Attribute[channel=0], zoneId=Z, locale=en_US]"), DataType.KEYWORD, equalTo(new BytesRef(expectedMonthName)) - ) + ).withStaticConfiguration() ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java index 10eac16848f94..ef5d6c2a18950 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java @@ -42,7 +42,7 @@ public static Iterable parameters() { matchesPattern("LiteralsEvaluator\\[lit=.*]"), DataType.DATETIME, equalTo(TestCaseSupplier.TEST_CONFIGURATION.now().toInstant().toEpochMilli()) - ) + ).withStaticConfiguration() ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ScalbTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ScalbTests.java index b17d539a32d89..1de02139f5cea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ScalbTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ScalbTests.java @@ -94,26 +94,26 @@ private static TestCaseSupplier test( Number res, Iterable warns ) { - var supplier = new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(d, dType, "d"), new TestCaseSupplier.TypedData(scaleFactor, scaleType, "scaleFactor")), - Strings.format( - "Scalb%sEvaluator[d=%s, scaleFactor=Attribute[channel=1]]", - pascalize(scaleType), - cast(dType) - - ), - DataType.DOUBLE, - equalTo(res) - ); - for (var warn : warns) { - supplier = supplier.withWarning(warn); - } - var finalSupplier = supplier; - return new TestCaseSupplier( - Strings.format("<%s>, <%s>", dType.typeName(), scaleType.typeName()), - List.of(dType, scaleType), - () -> finalSupplier - ); + return new TestCaseSupplier(Strings.format("<%s>, <%s>", dType.typeName(), scaleType.typeName()), List.of(dType, scaleType), () -> { + var supplier = new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(d, dType, "d"), + new TestCaseSupplier.TypedData(scaleFactor, scaleType, "scaleFactor") + ), + Strings.format( + "Scalb%sEvaluator[d=%s, scaleFactor=Attribute[channel=1]]", + pascalize(scaleType), + cast(dType) + + ), + DataType.DOUBLE, + equalTo(res) + ); + for (var warn : warns) { + supplier = supplier.withWarning(warn); + } + return supplier; + }); } private static String cast(DataType from) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index 81047c0e7dc80..894b1249426ec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -91,7 +91,8 @@ private static void suppliers(List suppliers, String name, Dat values.add(new TestCaseSupplier.TypedData(new BytesRef(value), type, "0")); String expectedValue = value.toLowerCase(EsqlTestUtils.TEST_CFG.locale()); - return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(new BytesRef(expectedValue))); + return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(new BytesRef(expectedValue))) + .withStaticConfiguration(); })); suppliers.add(new TestCaseSupplier(name + " mv", List.of(type), () -> { List values = new ArrayList<>(); @@ -101,7 +102,7 @@ private static void suppliers(List suppliers, String name, Dat values.add(new TestCaseSupplier.TypedData(strings.stream().map(BytesRef::new).toList(), type, "0")); List expectedValue = strings.stream().map(s -> new BytesRef(s.toLowerCase(EsqlTestUtils.TEST_CFG.locale()))).toList(); - return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue)); + return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue)).withStaticConfiguration(); })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index 946d8cca22f46..c3ab1ae0c2614 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -91,7 +91,8 @@ private static void supplier(List suppliers, String name, Data values.add(new TestCaseSupplier.TypedData(new BytesRef(value), type, "0")); String expectedValue = value.toUpperCase(EsqlTestUtils.TEST_CFG.locale()); - return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(new BytesRef(expectedValue))); + return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(new BytesRef(expectedValue))) + .withStaticConfiguration(); })); suppliers.add(new TestCaseSupplier(name + " mv", List.of(type), () -> { List values = new ArrayList<>(); @@ -101,7 +102,7 @@ private static void supplier(List suppliers, String name, Data values.add(new TestCaseSupplier.TypedData(strings.stream().map(BytesRef::new).toList(), type, "0")); List expectedValue = strings.stream().map(s -> new BytesRef(s.toUpperCase(EsqlTestUtils.TEST_CFG.locale()))).toList(); - return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue)); + return new TestCaseSupplier.TestCase(values, expectedToString, type, equalTo(expectedValue)).withStaticConfiguration(); })); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/240_timezone.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/240_timezone.yml new file mode 100644 index 0000000000000..c877a65effa19 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/240_timezone.yml @@ -0,0 +1,188 @@ +--- +setup: + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ ] + capabilities: [ global_timezone_parameter ] + test_runner_features: [ capabilities ] + reason: "Check timezone capability" + - do: + indices.create: + index: logs + body: + mappings: + properties: + "@timestamp": + type: date + + - do: + bulk: + index: tzs + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2025-05-31T00:00:00Z" } + - { "index": { } } + - { "@timestamp": "2025-05-31T01:00:00Z" } + - { "index": { } } + - { "@timestamp": "2025-05-31T22:00:00Z" } + - { "index": { } } + - { "@timestamp": "2025-05-31T23:00:00Z" } + +--- +Default UTC: + - do: + esql.query: + body: + query: | + FROM tzs + | EVAL + hour=DATE_EXTRACT("hour_of_day", @timestamp), + day_name=DAY_NAME(@timestamp), + month_name=MONTH_NAME(@timestamp) + | SORT @timestamp ASC + | LIMIT 5 + + - length: { columns: 4 } + - match: { columns.0.name: "@timestamp" } + - match: { columns.1.name: "hour" } + - match: { columns.2.name: "day_name" } + - match: { columns.3.name: "month_name" } + + - length: { values: 4 } + - match: { values.0.0: "2025-05-31T00:00:00.000Z" } + - match: { values.0.1: 0 } + - match: { values.0.2: "Saturday" } + - match: { values.0.3: "May" } + - match: { values.1.0: "2025-05-31T01:00:00.000Z" } + - match: { values.1.1: 1 } + - match: { values.1.2: "Saturday" } + - match: { values.1.3: "May" } + - match: { values.2.0: "2025-05-31T22:00:00.000Z" } + - match: { values.2.1: 22 } + - match: { values.2.2: "Saturday" } + - match: { values.2.3: "May" } + - match: { values.3.0: "2025-05-31T23:00:00.000Z" } + - match: { values.3.1: 23 } + - match: { values.3.2: "Saturday" } + - match: { values.3.3: "May" } + +--- +Paris tz with SET: + - do: + esql.query: + body: + query: | + SET time_zone="Europe/Paris"; + FROM tzs + | EVAL + hour=DATE_EXTRACT("hour_of_day", @timestamp), + day_name=DAY_NAME(@timestamp), + month_name=MONTH_NAME(@timestamp) + | SORT @timestamp ASC + | LIMIT 5 + + - length: { columns: 4 } + - match: { columns.0.name: "@timestamp" } + - match: { columns.1.name: "hour" } + - match: { columns.2.name: "day_name" } + - match: { columns.3.name: "month_name" } + + - length: { values: 4 } + - match: { values.0.0: "2025-05-31T00:00:00.000Z" } + - match: { values.0.1: 2 } + - match: { values.0.2: "Saturday" } + - match: { values.0.3: "May" } + - match: { values.1.0: "2025-05-31T01:00:00.000Z" } + - match: { values.1.1: 3 } + - match: { values.1.2: "Saturday" } + - match: { values.1.3: "May" } + - match: { values.2.0: "2025-05-31T22:00:00.000Z" } + - match: { values.2.1: 0 } + - match: { values.2.2: "Sunday" } + - match: { values.2.3: "June" } + - match: { values.3.0: "2025-05-31T23:00:00.000Z" } + - match: { values.3.1: 1 } + - match: { values.3.2: "Sunday" } + - match: { values.3.3: "June" } + +--- +Paris tz with request parameter: + - do: + esql.query: + body: + time_zone: "Europe/Paris" + query: | + FROM tzs + | EVAL + hour=DATE_EXTRACT("hour_of_day", @timestamp), + day_name=DAY_NAME(@timestamp), + month_name=MONTH_NAME(@timestamp) + | SORT @timestamp ASC + | LIMIT 5 + + - length: { columns: 4 } + - match: { columns.0.name: "@timestamp" } + - match: { columns.1.name: "hour" } + - match: { columns.2.name: "day_name" } + - match: { columns.3.name: "month_name" } + + - length: { values: 4 } + - match: { values.0.0: "2025-05-31T00:00:00.000Z" } + - match: { values.0.1: 2 } + - match: { values.0.2: "Saturday" } + - match: { values.0.3: "May" } + - match: { values.1.0: "2025-05-31T01:00:00.000Z" } + - match: { values.1.1: 3 } + - match: { values.1.2: "Saturday" } + - match: { values.1.3: "May" } + - match: { values.2.0: "2025-05-31T22:00:00.000Z" } + - match: { values.2.1: 0 } + - match: { values.2.2: "Sunday" } + - match: { values.2.3: "June" } + - match: { values.3.0: "2025-05-31T23:00:00.000Z" } + - match: { values.3.1: 1 } + - match: { values.3.2: "Sunday" } + - match: { values.3.3: "June" } + +--- +Set overrides request parameter: + - do: + esql.query: + body: + time_zone: "Europe/Paris" + query: | + SET time_zone="+04:00"; + FROM tzs + | EVAL + hour=DATE_EXTRACT("hour_of_day", @timestamp), + day_name=DAY_NAME(@timestamp), + month_name=MONTH_NAME(@timestamp) + | SORT @timestamp ASC + | LIMIT 5 + + - length: { columns: 4 } + - match: { columns.0.name: "@timestamp" } + - match: { columns.1.name: "hour" } + - match: { columns.2.name: "day_name" } + - match: { columns.3.name: "month_name" } + + - length: { values: 4 } + - match: { values.0.0: "2025-05-31T00:00:00.000Z" } + - match: { values.0.1: 4 } + - match: { values.0.2: "Saturday" } + - match: { values.0.3: "May" } + - match: { values.1.0: "2025-05-31T01:00:00.000Z" } + - match: { values.1.1: 5 } + - match: { values.1.2: "Saturday" } + - match: { values.1.3: "May" } + - match: { values.2.0: "2025-05-31T22:00:00.000Z" } + - match: { values.2.1: 2 } + - match: { values.2.2: "Sunday" } + - match: { values.2.3: "June" } + - match: { values.3.0: "2025-05-31T23:00:00.000Z" } + - match: { values.3.1: 3 } + - match: { values.3.2: "Sunday" } + - match: { values.3.3: "June" }