Skip to content

fix: select_spark_version(latest=True) no longer implicitly filters to scala 2.12 #1542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mittalpk wants to merge 1 commit into
base: main
Choose a base branch
from
+36 −3
Open

fix: select_spark_version(latest=True) no longer implicitly filters to scala 2.12 #1542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions databricks/sdk/mixins/compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def select_spark_version(
ml: bool = False,
genomics: bool = False,
gpu: bool = False,
scala: str = "2.12",
scala: str = "",
spark_version: str = None,
photon: bool = False,
graviton: bool = False,
Expand All @@ -95,17 +95,19 @@ def select_spark_version(
:param genomics: bool
:param gpu: bool
:param scala: str
Scala version to filter on, e.g. "2.12" or "2.13". Leave empty (the default) to
consider every Scala version, matching the Go SDK's zero-value behavior.
:param spark_version: str
:param photon: bool
:param graviton: bool

:returns: `spark_version` compatible string
"""
# Logic ported from https://github.com/databricks/databricks-sdk-go/blob/main/service/compute/spark_version.go
# Logic ported from https://github.com/databricks/databricks-sdk-go/blob/main/service/compute/ext_spark_version.go
versions = []
sv = self.spark_versions()
for version in sv.versions:
if "-scala" + scala not in version.key:
if scala and "-scala" + scala not in version.key:
continue
matches = (
("apache-spark-" not in version.key)
Expand Down
31 changes: 31 additions & 0 deletions tests/test_compute_mixins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
import json

import pytest

from databricks.sdk.mixins.compute import SemVer

SPARK_VERSIONS_RESPONSE = {
"versions": [
{"key": "16.4.x-scala2.12", "name": "16.4"},
{"key": "18.2.x-scala2.13", "name": "18.2"},
]
}


@pytest.mark.parametrize(
"given,expected",
Expand Down Expand Up @@ -40,3 +49,25 @@ def test_sorting_semver():
SemVer(1, 0, 0),
SemVer(12, 0, 0),
]


def test_select_spark_version_latest_ignores_scala_by_default(w, requests_mock):
# Regression test for https://github.com/databricks/databricks-sdk-py/issues/1487:
# select_spark_version(latest=True) implicitly filtered to the "2.12" default
# scala version before picking the latest, instead of considering every scala
# version like the Go SDK it's ported from does.
requests_mock.get(
"http://localhost/api/2.1/clusters/spark-versions",
text=json.dumps(SPARK_VERSIONS_RESPONSE),
)

assert w.clusters.select_spark_version(latest=True) == "18.2.x-scala2.13"


def test_select_spark_version_latest_still_honors_an_explicit_scala(w, requests_mock):
requests_mock.get(
"http://localhost/api/2.1/clusters/spark-versions",
text=json.dumps(SPARK_VERSIONS_RESPONSE),
)

assert w.clusters.select_spark_version(latest=True, scala="2.12") == "16.4.x-scala2.12"
Loading