upgrade pacakge dependencies (#921)

Signed-off-by: cching95 <chloe.ching@shell.com>

Author: cching95

environment.yml

@@ -19,8 +19,8 @@ channels:
     - defaults
 dependencies:
     - python>=3.9,<3.13
-    - importlib-metadata>=7.0.0
-    - jinja2>=3.1.5
+    - importlib-metadata>=7.0.0,<9.0.0
+    - jinja2>=3.1.5,<4.0.0
     - pytest==7.4.0
     - pytest-mock==3.11.1
     - pytest-cov==4.1.0
@@ -40,8 +40,10 @@ dependencies:
     - httpx>=0.24.1,<1.0.0
     - pyspark>=3.3.0,<3.6.0
     - delta-spark>=2.2.0,<3.3.0
-    - grpcio>=1.48.1
-    - grpcio-status>=1.48.1
+    - pyarrow>=14.0.1,<17.0.0
+    - libarrow>=14.0.1,<17.0.0
+    - grpcio>=1.48.1,<1.63.0
+    - grpcio-status>=1.48.1,<1.63.0
     - googleapis-common-protos>=1.56.4
     - openjdk>=11.0.15,<12.0.0
     - openai>=1.59.0,<2.0.0
@@ -70,8 +72,11 @@ dependencies:
     - great-expectations>=0.18.8,<1.0.0
     - statsmodels>=0.14.1,<0.15.0
     - pmdarima>=2.0.4
-    - protobuf>=5.28.2,<5.29.0
     - pip:
+          # protobuf installed via pip to avoid libabseil conflicts with conda libarrow
+          - protobuf>=5.29.0,<5.30.0
+          # pyarrow constraint must match conda version to avoid upgrades
+          - pyarrow>=14.0.1,<17.0.0
           - databricks-sdk>=0.59.0,<1.0.0
           - dependency-injector>=4.41.0,<5.0.0
           - azure-functions>=1.20.0,<2.0.0

setup.py

@@ -28,19 +28,19 @@
 long_description = (here / "PYPI-README.md").read_text()
 
 INSTALL_REQUIRES = [
-    "databricks-sql-connector>=3.1.0,<3.7.0",
+    "databricks-sql-connector>=3.6.0,<3.7.0",
     "pyarrow>=14.0.1,<17.0.0",
-    "azure-identity>=1.12.0,<2.0.0",
+    "azure-identity>=1.20.0,<2.0.0",
     "pandas>=2.0.1,<2.3.0",
     "jinja2>=3.1.5,<4.0.0",
-    "importlib_metadata>=7.0.0,<8.0.0",
+    "importlib_metadata>=7.0.0,<9.0.0",
     "semver>=3.0.0,<4.0.0",
     "xlrd>=2.0.1,<3.0.0",
-    "grpcio>=1.48.1",
-    "grpcio-status>=1.48.1",
+    "grpcio>=1.48.1,<1.63.0",
+    "grpcio-status>=1.48.1,<1.63.0",
     "googleapis-common-protos>=1.56.4",
-    "pydantic>=2.6.0,<3.0.0",
-    "protobuf>=5.28.2,<5.29.0",
+    "pydantic>=2.10.0,<3.0.0",
+    "protobuf>=5.29.0,<5.30.0",
 ]
 
 LANGCHAIN_PACKAGES = [

src/api/requirements.txt

@@ -4,21 +4,22 @@ fastapi==0.115.6
 pydantic==2.10.0
 # turbodbc==4.11.0
 pyodbc==5.2.0
-importlib_metadata>=7.0.0
+importlib_metadata>=7.0.0,<9.0.0
 databricks-sql-connector==3.6.0
 azure-identity==1.20.0
 oauthlib>=3.2.2
 pandas>=2.0.1,<2.3.0
 numpy==1.26.4
+pyarrow>=14.0.1,<17.0.0
 jinja2==3.1.5
 pytz==2024.2
 semver==3.0.2
 xlrd==2.0.1
 packaging==24.2
-grpcio>=1.48.1
-grpcio-status>=1.48.1
+grpcio>=1.48.1,<1.63.0
+grpcio-status>=1.48.1,<1.63.0
 googleapis-common-protos>=1.56.4
-protobuf>=5.28.2,<5.29.0
+protobuf>=5.29.0,<5.30.0
 langchain>=0.3.27,<0.4.0
 langchain-community>=0.3.27,<0.4.0
 langchain-core>=0.3.28,<0.4.0