Skip to content

Commit a7973fd

Browse files
authored
feat: External UDF support STAGE_LOCATION param (#18833)
* feat: External UDF support STAGE_LOCATION param * chore: codefmt * ci: add test for external udf stage location param * chore: fix parse test * chore; fix e2e test * chore: fix e2e test * feat: Added Expr::StageLocation for passing StageLocation as a Function parameter * chore: codefmt * chore: add unit test for TableDataType::StageLocation
1 parent 1371711 commit a7973fd

File tree

50 files changed

+600
-74
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+600
-74
lines changed

.github/actions/test_stateless_cluster_linux/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ runs:
1313
- name: Start UDF Server
1414
shell: bash
1515
run: |
16-
pip install databend-udf>=0.2.6
16+
pip install databend-udf>=0.2.7
1717
python3 tests/udf/udf_server.py &
1818
sleep 2
1919

.github/actions/test_stateless_cluster_macos/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ runs:
1313
- name: Start UDF Server
1414
shell: bash
1515
run: |
16-
pip install databend-udf>=0.2.6
16+
pip install databend-udf>=0.2.7
1717
python3 tests/udf/udf_server.py &
1818
sleep 2
1919

.github/actions/test_stateless_standalone_linux/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ runs:
1313
- name: Start UDF Server
1414
shell: bash
1515
run: |
16-
pip install databend-udf>=0.2.6
16+
pip install databend-udf>=0.2.7
1717
python3 tests/udf/udf_server.py &
1818
sleep 2
1919

.github/actions/test_stateless_standalone_macos/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ runs:
1313
- name: Start UDF Server
1414
shell: bash
1515
run: |
16-
pip install databend-udf>=0.2.6
16+
pip install databend-udf>=0.2.7
1717
python3 tests/udf/udf_server.py &
1818
sleep 2
1919

.github/workflows/reuse.sqllogic.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@ jobs:
9292
- uses: actions/checkout@v4
9393
- name: Start UDF Server
9494
run: |
95-
pip install databend-udf>=0.2.6
95+
docker run -d --name minio -p 9000:9000 -p 9001:9001 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/minio/minio server /data --console-address ":9001"
96+
pip install databend-udf>=0.2.7
9697
python3 tests/udf/udf_server.py &
9798
sleep 2
9899
- uses: ./.github/actions/test_sqllogic_standalone_linux

src/meta/app/src/principal/user_defined_function.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ pub struct UDFServer {
3333
pub handler: String,
3434
pub headers: BTreeMap<String, String>,
3535
pub language: String,
36+
pub arg_names: Vec<String>,
3637
pub arg_types: Vec<DataType>,
3738
pub return_type: DataType,
3839
pub immutable: Option<bool>,
@@ -168,6 +169,7 @@ impl UserDefinedFunction {
168169
handler: &str,
169170
headers: &BTreeMap<String, String>,
170171
language: &str,
172+
arg_names: Vec<String>,
171173
arg_types: Vec<DataType>,
172174
return_type: DataType,
173175
description: &str,
@@ -181,6 +183,7 @@ impl UserDefinedFunction {
181183
handler: handler.to_string(),
182184
headers: headers.clone(),
183185
language: language.to_string(),
186+
arg_names,
184187
arg_types,
185188
return_type,
186189
immutable,
@@ -237,6 +240,7 @@ impl Display for UDFDefinition {
237240
}
238241
UDFDefinition::UDFServer(UDFServer {
239242
address,
243+
arg_names,
240244
arg_types,
241245
return_type,
242246
handler,
@@ -249,6 +253,9 @@ impl Display for UDFDefinition {
249253
write!(f, ", ")?;
250254
}
251255
write!(f, "{item}")?;
256+
if !arg_names.is_empty() {
257+
write!(f, " {}", arg_names[i])?;
258+
}
252259
}
253260
write!(f, ") RETURNS {return_type} LANGUAGE {language}")?;
254261
if let Some(immutable) = immutable {

src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ impl FromToProto for ex::TableDataType {
312312
Dt24::VectorT(v) => {
313313
ex::TableDataType::Vector(ex::types::VectorDataType::from_pb(v)?)
314314
}
315+
Dt24::StageLocationT(_) => ex::TableDataType::StageLocation,
315316
};
316317
Ok(x)
317318
}
@@ -380,6 +381,7 @@ impl FromToProto for ex::TableDataType {
380381
let x = v.to_pb()?;
381382
new_pb_dt24(Dt24::VectorT(x))
382383
}
384+
TableDataType::StageLocation => new_pb_dt24(Dt24::StageLocationT(pb::Empty {})),
383385
};
384386
Ok(x)
385387
}

src/meta/proto-conv/src/udf_from_to_protobuf_impl.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ impl FromToProto for mt::UDFServer {
7777
headers: p.headers,
7878
language: p.language,
7979
immutable: p.immutable,
80+
arg_names: p.arg_names,
8081
})
8182
}
8283

@@ -112,6 +113,7 @@ impl FromToProto for mt::UDFServer {
112113
arg_types,
113114
return_type: Some(return_type),
114115
immutable: self.immutable,
116+
arg_names: self.arg_names.clone(),
115117
})
116118
}
117119
}

src/meta/proto-conv/src/util.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
181181
(149, "2025-09-24: Add: add AutoIncrement name and display on TableField"),
182182
(150, "2025-09-26: Add: RoleInfo::comment"),
183183
(151, "2025-09-28: Add: TableMeta::RowAccessPolicyColumnMap store policy name and column id"),
184+
(152, "2025-10-14: Add: TableDataType::StageLocation and UDFServer add arg_names"),
184185
// Dear developer:
185186
// If you're gonna add a new metadata version, you'll have to add a test for it.
186187
// You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,3 +143,4 @@ mod v148_virtual_schema;
143143
mod v149_field_auto_increment;
144144
mod v150_role_comment;
145145
mod v151_row_access_column_map;
146+
mod v152_external_udf;

0 commit comments

Comments
 (0)