feat: External UDF support STAGE_LOCATION param

.github/actions/test_stateless_cluster_linux/action.yml

@@ -13,7 +13,7 @@ runs:
     - name: Start UDF Server
       shell: bash
       run: |
-        pip install databend-udf>=0.2.6
+        pip install databend-udf>=0.2.7
         python3 tests/udf/udf_server.py &
         sleep 2
 

.github/actions/test_stateless_cluster_macos/action.yml

@@ -13,7 +13,7 @@ runs:
     - name: Start UDF Server
       shell: bash
       run: |
-        pip install databend-udf>=0.2.6
+        pip install databend-udf>=0.2.7
         python3 tests/udf/udf_server.py &
         sleep 2
 

.github/actions/test_stateless_standalone_linux/action.yml

@@ -13,7 +13,7 @@ runs:
     - name: Start UDF Server
       shell: bash
       run: |
-        pip install databend-udf>=0.2.6
+        pip install databend-udf>=0.2.7
         python3 tests/udf/udf_server.py &
         sleep 2
 

.github/actions/test_stateless_standalone_macos/action.yml

@@ -13,7 +13,7 @@ runs:
     - name: Start UDF Server
       shell: bash
       run: |
-        pip install databend-udf>=0.2.6
+        pip install databend-udf>=0.2.7
         python3 tests/udf/udf_server.py &
         sleep 2
 

.github/workflows/reuse.sqllogic.yml

@@ -92,7 +92,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Start UDF Server
         run: |
-          pip install databend-udf>=0.2.6
+          docker run -d --name minio -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/minio/minio server /data --console-address ":9001"
+          pip install databend-udf>=0.2.7
           python3 tests/udf/udf_server.py &
           sleep 2
       - uses: ./.github/actions/test_sqllogic_standalone_linux

src/meta/app/src/principal/user_defined_function.rs

@@ -33,6 +33,7 @@ pub struct UDFServer {
     pub handler: String,
     pub headers: BTreeMap<String, String>,
     pub language: String,
+    pub arg_names: Vec<String>,
     pub arg_types: Vec<DataType>,
     pub return_type: DataType,
     pub immutable: Option<bool>,
@@ -168,6 +169,7 @@ impl UserDefinedFunction {
         handler: &str,
         headers: &BTreeMap<String, String>,
         language: &str,
+        arg_names: Vec<String>,
         arg_types: Vec<DataType>,
         return_type: DataType,
         description: &str,
@@ -181,6 +183,7 @@ impl UserDefinedFunction {
                 handler: handler.to_string(),
                 headers: headers.clone(),
                 language: language.to_string(),
+                arg_names,
                 arg_types,
                 return_type,
                 immutable,
@@ -237,6 +240,7 @@ impl Display for UDFDefinition {
             }
             UDFDefinition::UDFServer(UDFServer {
                 address,
+                arg_names,
                 arg_types,
                 return_type,
                 handler,
@@ -249,6 +253,9 @@ impl Display for UDFDefinition {
                         write!(f, ", ")?;
                     }
                     write!(f, "{item}")?;
+                    if !arg_names.is_empty() {
+                        write!(f, " {}", arg_names[i])?;
+                    }
                 }
                 write!(f, ") RETURNS {return_type} LANGUAGE {language}")?;
                 if let Some(immutable) = immutable {

src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs

@@ -312,6 +312,7 @@ impl FromToProto for ex::TableDataType {
                     Dt24::VectorT(v) => {
                         ex::TableDataType::Vector(ex::types::VectorDataType::from_pb(v)?)
                     }
+                    Dt24::StageLocationT(_) => ex::TableDataType::StageLocation,
                 };
                 Ok(x)
             }
@@ -380,6 +381,7 @@ impl FromToProto for ex::TableDataType {
                 let x = v.to_pb()?;
                 new_pb_dt24(Dt24::VectorT(x))
             }
+            TableDataType::StageLocation => new_pb_dt24(Dt24::StageLocationT(pb::Empty {})),
         };
         Ok(x)
     }

src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs

@@ -77,6 +77,7 @@ impl FromToProto for mt::UDFServer {
             headers: p.headers,
             language: p.language,
             immutable: p.immutable,
+            arg_names: p.arg_names,
         })
     }
 
@@ -112,6 +113,7 @@ impl FromToProto for mt::UDFServer {
             arg_types,
             return_type: Some(return_type),
             immutable: self.immutable,
+            arg_names: self.arg_names.clone(),
         })
     }
 }

src/meta/proto-conv/src/util.rs

@@ -181,6 +181,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (149, "2025-09-24: Add: add AutoIncrement name and display on TableField"),
     (150, "2025-09-26: Add: RoleInfo::comment"),
     (151, "2025-09-28: Add: TableMeta::RowAccessPolicyColumnMap store policy name and column id"),
+    (152, "2025-10-14: Add: TableDataType::StageLocation and UDFServer add arg_names"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -143,3 +143,4 @@ mod v148_virtual_schema;
 mod v149_field_auto_increment;
 mod v150_role_comment;
 mod v151_row_access_column_map;
+mod v152_external_udf;

src/meta/proto-conv/tests/it/proto_conv.rs

@@ -311,6 +311,7 @@ fn new_udf_server() -> databend_common_meta_app::principal::UDFServer {
             ("X-Api-Version".to_string(), "11".to_string()),
         ]),
         language: "python".to_string(),
+        arg_names: vec![],
         arg_types: vec![DataType::String],
         return_type: DataType::Boolean,
         immutable: None,

src/meta/proto-conv/tests/it/v058_udf.rs

@@ -55,6 +55,7 @@ fn test_decode_v57_udf() -> anyhow::Result<()> {
             handler: "plus_int_py".to_string(),
             headers: BTreeMap::default(),
             language: "python".to_string(),
+            arg_names: vec![],
             arg_types: vec![
                 DataType::Number(NumberDataType::Int32),
                 DataType::Number(NumberDataType::Int32),

src/meta/proto-conv/tests/it/v079_udf_created_on.rs

@@ -58,6 +58,7 @@ fn test_decode_v79_udf_python() -> anyhow::Result<()> {
             handler: "plus_int_py".to_string(),
             headers: BTreeMap::default(),
             language: "python".to_string(),
+            arg_names: vec![],
             arg_types: vec![
                 DataType::Number(NumberDataType::Int32),
                 DataType::Number(NumberDataType::Int32),

src/meta/proto-conv/tests/it/v081_udf_script.rs

@@ -59,6 +59,7 @@ fn test_decode_v81_udf_python() -> anyhow::Result<()> {
             handler: "plus_int_py".to_string(),
             headers: BTreeMap::default(),
             language: "python".to_string(),
+            arg_names: vec![],
             arg_types: vec![
                 DataType::Number(NumberDataType::Int32),
                 DataType::Number(NumberDataType::Int32),

src/meta/proto-conv/tests/it/v124_udf_server_headers.rs

@@ -47,6 +47,7 @@ fn test_decode_v124_udf_server_headers() -> anyhow::Result<()> {
             ("X-Api-Version".to_string(), "11".to_string()),
         ]),
         language: "python".to_string(),
+        arg_names: vec![],
         arg_types: vec![DataType::String],
         return_type: DataType::Boolean,
         immutable: None,

src/meta/proto-conv/tests/it/v135_udf_immutable.rs

@@ -60,6 +60,7 @@ fn test_decode_v135_udf_server() -> anyhow::Result<()> {
                 ("X-Api-Version".to_string(), "11".to_string()),
             ]),
             language: "python".to_string(),
+            arg_names: vec![],
             arg_types: vec![DataType::String],
             return_type: DataType::Boolean,
             immutable: Some(true),

src/meta/proto-conv/tests/it/v152_external_udf.rs

@@ -0,0 +1,80 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+
+use chrono::DateTime;
+use chrono::Utc;
+use databend_common_expression::types::DataType;
+use databend_common_expression::TableDataType;
+use databend_common_meta_app::principal::UDFDefinition;
+use databend_common_meta_app::principal::UDFServer;
+use databend_common_meta_app::principal::UserDefinedFunction;
+use fastrace::func_name;
+
+use crate::common;
+
+#[test]
+fn test_decode_v152_external_udf() -> anyhow::Result<()> {
+    let user_defined_function_v152 = vec![
+        10, 8, 116, 101, 115, 116, 95, 117, 100, 102, 18, 17, 105, 115, 32, 97, 32, 101, 120, 116,
+        101, 114, 110, 97, 108, 32, 117, 100, 102, 34, 123, 10, 21, 104, 116, 116, 112, 58, 47, 47,
+        49, 50, 55, 46, 48, 46, 48, 46, 49, 58, 56, 56, 56, 56, 18, 7, 105, 115, 101, 109, 112,
+        116, 121, 26, 6, 112, 121, 116, 104, 111, 110, 34, 10, 162, 3, 0, 160, 6, 152, 1, 168, 6,
+        24, 34, 10, 146, 2, 0, 160, 6, 152, 1, 168, 6, 24, 42, 10, 138, 2, 0, 160, 6, 152, 1, 168,
+        6, 24, 50, 19, 10, 13, 88, 45, 65, 112, 105, 45, 86, 101, 114, 115, 105, 111, 110, 18, 2,
+        49, 49, 50, 17, 10, 7, 88, 45, 84, 111, 107, 101, 110, 18, 6, 97, 98, 99, 49, 50, 51, 160,
+        6, 152, 1, 168, 6, 24, 42, 23, 49, 57, 55, 48, 45, 48, 49, 45, 48, 49, 32, 48, 48, 58, 48,
+        48, 58, 48, 48, 32, 85, 84, 67, 160, 6, 152, 1, 168, 6, 24,
+    ];
+
+    let want = || UserDefinedFunction {
+        name: "test_udf".to_string(),
+        description: "is a external udf".to_string(),
+        definition: UDFDefinition::UDFServer(UDFServer {
+            address: "http://127.0.0.1:8888".to_string(),
+            handler: "isempty".to_string(),
+            headers: BTreeMap::from([
+                ("X-Token".to_string(), "abc123".to_string()),
+                ("X-Api-Version".to_string(), "11".to_string()),
+            ]),
+            language: "python".to_string(),
+            arg_names: vec![],
+            arg_types: vec![DataType::StageLocation, DataType::String],
+            return_type: DataType::Boolean,
+            immutable: None,
+        }),
+        created_on: DateTime::<Utc>::default(),
+    };
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(
+        func_name!(),
+        user_defined_function_v152.as_slice(),
+        152,
+        want(),
+    )?;
+
+    Ok(())
+}
+
+#[test]
+fn test_decode_v152_data_type_stage_location() -> anyhow::Result<()> {
+    let table_data_type_v152 = vec![162, 3, 0, 160, 6, 152, 1, 168, 6, 24];
+
+    let want = || TableDataType::StageLocation;
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), table_data_type_v152.as_slice(), 152, want())?;
+
+    Ok(())
+}

src/meta/protos/proto/datatype.proto

@@ -69,6 +69,7 @@ message DataType {
     Empty    interval_t    = 49;
     Vector   vector_t      = 50;
     uint64   opaque_t      = 51;
+    Empty    stage_location_t = 52;
   }
 }
 

src/meta/protos/proto/udf.proto

@@ -38,6 +38,7 @@ message UDFServer {
   DataType return_type = 5;
   map<string, string> headers = 6;
   optional bool immutable = 7;
+  repeated string arg_names = 8;
 }
 
 message UDFScript {

src/query/ast/src/ast/expr.rs

@@ -296,6 +296,10 @@ pub enum Expr {
     Placeholder {
         span: Span,
     },
+    StageLocation {
+        span: Span,
+        location: String,
+    },
 }
 
 impl Expr {
@@ -341,7 +345,8 @@ impl Expr {
             | Expr::PreviousDay { span, .. }
             | Expr::NextDay { span, .. }
             | Expr::Hole { span, .. }
-            | Expr::Placeholder { span } => *span,
+            | Expr::Placeholder { span }
+            | Expr::StageLocation { span, .. } => *span,
         }
     }
 
@@ -510,6 +515,7 @@ impl Expr {
             Expr::NextDay { span, date, .. } => merge_span(*span, date.whole_span()),
             Expr::Hole { span, .. } => *span,
             Expr::Placeholder { span } => *span,
+            Expr::StageLocation { span, .. } => *span,
         }
     }
 
@@ -906,6 +912,9 @@ impl Display for Expr {
                 Expr::Placeholder { .. } => {
                     write!(f, "?")?;
                 }
+                Expr::StageLocation { location, .. } => {
+                    write!(f, "@{location}")?;
+                }
             }
 
             if need_paren {
@@ -1201,6 +1210,7 @@ pub enum TypeName {
     Vector(u64),
     Nullable(Box<TypeName>),
     NotNull(Box<TypeName>),
+    StageLocation,
 }
 
 impl TypeName {
@@ -1333,6 +1343,9 @@ impl Display for TypeName {
             TypeName::Vector(dimension) => {
                 write!(f, "VECTOR({dimension})")?;
             }
+            TypeName::StageLocation => {
+                write!(f, "STAGE_LOCATION")?;
+            }
         }
         Ok(())
     }