Skip to content

Commit 4572507

Browse files
majin1102majin1102
committed
init
1 parent cad9869 commit 4572507

15 files changed

Lines changed: 847 additions & 4 deletions

File tree

‎amoro-ams/pom.xml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
<properties>
3535
<swagger-ui.version>5.17.14</swagger-ui.version>
36+
<lance-arrow-version>18.3.0</lance-arrow-version>
3637
<git-commit-id-plugin.fail-on-no-git-dir>false</git-commit-id-plugin.fail-on-no-git-dir>
3738
</properties>
3839

@@ -409,6 +410,13 @@
409410
<scope>test</scope>
410411
</dependency>
411412

413+
<dependency>
414+
<groupId>org.apache.amoro</groupId>
415+
<artifactId>amoro-format-lance</artifactId>
416+
<version>${project.version}</version>
417+
<scope>test</scope>
418+
</dependency>
419+
412420
<dependency>
413421
<groupId>org.apache.curator</groupId>
414422
<artifactId>curator-test</artifactId>
@@ -618,6 +626,66 @@
618626
</dependency>
619627
</dependencies>
620628
</profile>
629+
<profile>
630+
<id>support-lance-format</id>
631+
<dependencyManagement>
632+
<dependencies>
633+
<dependency>
634+
<groupId>org.apache.arrow</groupId>
635+
<artifactId>arrow-bom</artifactId>
636+
<version>${lance-arrow-version}</version>
637+
<type>pom</type>
638+
<scope>import</scope>
639+
</dependency>
640+
</dependencies>
641+
</dependencyManagement>
642+
<dependencies>
643+
<dependency>
644+
<groupId>org.apache.amoro</groupId>
645+
<artifactId>amoro-format-lance</artifactId>
646+
</dependency>
647+
<dependency>
648+
<groupId>org.apache.arrow</groupId>
649+
<artifactId>arrow-vector</artifactId>
650+
</dependency>
651+
<dependency>
652+
<groupId>org.apache.arrow</groupId>
653+
<artifactId>arrow-memory-netty</artifactId>
654+
</dependency>
655+
<dependency>
656+
<groupId>org.apache.arrow</groupId>
657+
<artifactId>arrow-c-data</artifactId>
658+
</dependency>
659+
<dependency>
660+
<groupId>org.apache.arrow</groupId>
661+
<artifactId>arrow-dataset</artifactId>
662+
</dependency>
663+
</dependencies>
664+
<build>
665+
<plugins>
666+
<plugin>
667+
<groupId>org.apache.maven.plugins</groupId>
668+
<artifactId>maven-enforcer-plugin</artifactId>
669+
<version>3.5.0</version>
670+
<executions>
671+
<execution>
672+
<id>enforce-arrow-dependency-convergence</id>
673+
<goals>
674+
<goal>enforce</goal>
675+
</goals>
676+
<phase>validate</phase>
677+
<configuration>
678+
<rules>
679+
<DependencyConvergence></DependencyConvergence>
680+
</rules>
681+
<fail>true</fail>
682+
</configuration>
683+
</execution>
684+
</executions>
685+
</plugin>
686+
</plugins>
687+
</build>
688+
</profile>
621689
<profile>
622690
<id>support-all-formats</id>
623691
<dependencies>
@@ -629,6 +697,10 @@
629697
<groupId>org.apache.amoro</groupId>
630698
<artifactId>amoro-format-hudi</artifactId>
631699
</dependency>
700+
<dependency>
701+
<groupId>org.apache.amoro</groupId>
702+
<artifactId>amoro-format-lance</artifactId>
703+
</dependency>
632704
</dependencies>
633705
</profile>
634706
<profile>

‎amoro-ams/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,18 @@ public class CatalogBuilder {
4444
ImmutableMap.of(
4545
CATALOG_TYPE_HADOOP,
4646
Sets.newHashSet(
47-
TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI),
47+
TableFormat.ICEBERG,
48+
TableFormat.MIXED_ICEBERG,
49+
TableFormat.PAIMON,
50+
TableFormat.HUDI,
51+
TableFormat.LANCE),
4852
CATALOG_TYPE_FILESYSTEM,
4953
Sets.newHashSet(
50-
TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON, TableFormat.HUDI),
54+
TableFormat.ICEBERG,
55+
TableFormat.MIXED_ICEBERG,
56+
TableFormat.PAIMON,
57+
TableFormat.HUDI,
58+
TableFormat.LANCE),
5159
CATALOG_TYPE_GLUE,
5260
Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
5361
CATALOG_TYPE_REST,

‎amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static org.apache.amoro.TableFormat.HUDI;
2222
import static org.apache.amoro.TableFormat.ICEBERG;
23+
import static org.apache.amoro.TableFormat.LANCE;
2324
import static org.apache.amoro.TableFormat.MIXED_HIVE;
2425
import static org.apache.amoro.TableFormat.MIXED_ICEBERG;
2526
import static org.apache.amoro.TableFormat.PAIMON;
@@ -161,6 +162,18 @@ public class CatalogController {
161162
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_HADOOP, PAIMON));
162163
VALIDATE_CATALOGS.add(
163164
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_S3, PAIMON));
165+
VALIDATE_CATALOGS.add(
166+
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
167+
VALIDATE_CATALOGS.add(
168+
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
169+
VALIDATE_CATALOGS.add(
170+
CatalogDescriptor.of(CATALOG_TYPE_FILESYSTEM, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
171+
VALIDATE_CATALOGS.add(
172+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_LOCAL, LANCE));
173+
VALIDATE_CATALOGS.add(
174+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_S3, LANCE));
175+
VALIDATE_CATALOGS.add(
176+
CatalogDescriptor.of(CATALOG_TYPE_HADOOP, STORAGE_CONFIGS_VALUE_TYPE_OSS, LANCE));
164177
VALIDATE_CATALOGS.add(
165178
CatalogDescriptor.of(CATALOG_TYPE_GLUE, STORAGE_CONFIGS_VALUE_TYPE_S3, ICEBERG));
166179
VALIDATE_CATALOGS.add(

‎amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public AmoroTable<?> loadTable(String database, String table) {
120120
TableFormat.MIXED_ICEBERG,
121121
TableFormat.ICEBERG,
122122
TableFormat.PAIMON,
123-
TableFormat.HUDI)
123+
TableFormat.HUDI,
124+
TableFormat.LANCE)
124125
.map(
125126
formatCatalog -> {
126127
try {
@@ -147,7 +148,8 @@ public List<TableIDWithFormat> listTables(String database) {
147148
TableFormat.MIXED_ICEBERG,
148149
TableFormat.ICEBERG,
149150
TableFormat.PAIMON,
150-
TableFormat.HUDI
151+
TableFormat.HUDI,
152+
TableFormat.LANCE
151153
};
152154

153155
Map<String, TableFormat> tableNameToFormat = Maps.newHashMap();

‎amoro-common/src/main/java/org/apache/amoro/TableFormat.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ public final class TableFormat implements Serializable {
4242
/** Open-source table formats */
4343
public static final TableFormat ICEBERG = register("ICEBERG");
4444

45+
public static final TableFormat LANCE = register("LANCE");
4546
public static final TableFormat MIXED_ICEBERG = register("MIXED_ICEBERG");
4647
public static final TableFormat MIXED_HIVE = register("MIXED_HIVE");
4748
public static final TableFormat PAIMON = register("PAIMON");

‎amoro-format-lance/pom.xml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>org.apache.amoro</groupId>
24+
<artifactId>amoro-parent</artifactId>
25+
<version>0.9-SNAPSHOT</version>
26+
<relativePath>../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>amoro-format-lance</artifactId>
30+
<name>Amoro Project Lance Format</name>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>org.apache.amoro</groupId>
35+
<artifactId>amoro-common</artifactId>
36+
</dependency>
37+
38+
<!-- Lance format Java API -->
39+
<dependency>
40+
<groupId>org.lance</groupId>
41+
<artifactId>lance-core</artifactId>
42+
<version>2.0.0-beta.8</version>
43+
</dependency>
44+
45+
<!-- Lance namespace interfaces and models -->
46+
<dependency>
47+
<groupId>org.lance</groupId>
48+
<artifactId>lance-namespace-core</artifactId>
49+
<version>0.3.2</version>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.lance</groupId>
53+
<artifactId>lance-namespace-apache-client</artifactId>
54+
<version>0.3.2</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.iceberg</groupId>
58+
<artifactId>iceberg-aws</artifactId>
59+
</dependency>
60+
<dependency>
61+
<groupId>org.apache.iceberg</groupId>
62+
<artifactId>iceberg-aliyun</artifactId>
63+
</dependency>
64+
</dependencies>
65+
66+
</project>

‎amoro-format-lance/src/main/java/org/apache/amoro/formats/lance/LanceCatalogFactory.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.amoro.formats.lance;
20+
21+
import org.apache.amoro.FormatCatalog;
22+
import org.apache.amoro.FormatCatalogFactory;
23+
import org.apache.amoro.TableFormat;
24+
import org.apache.amoro.table.TableMetaStore;
25+
26+
import java.util.HashMap;
27+
import java.util.Map;
28+
29+
/** Lance format catalog factory. */
30+
public class LanceCatalogFactory implements FormatCatalogFactory {
31+
32+
/** TableFormat instance for Lance. */
33+
public static final TableFormat LANCE = TableFormat.register("LANCE");
34+
35+
@Override
36+
public FormatCatalog create(
37+
String catalogName,
38+
String metastoreType,
39+
Map<String, String> properties,
40+
TableMetaStore metaStore) {
41+
return new LanceDirectoryV1Catalog(catalogName, properties);
42+
}
43+
44+
@Override
45+
public TableFormat format() {
46+
return LANCE;
47+
}
48+
49+
@Override
50+
public Map<String, String> convertCatalogProperties(
51+
String catalogName, String metastoreType, Map<String, String> unifiedCatalogProperties) {
52+
if (unifiedCatalogProperties == null) {
53+
return null;
54+
}
55+
56+
return new HashMap<>(unifiedCatalogProperties);
57+
}
58+
}

0 commit comments

Comments
 (0)