Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9272f5a
[Subtask]: add AmsAssignService to implement balanced bucket allocati…
Nov 10, 2025
8464424
[Subtask]: Add a registration function for table allocation in master…
Nov 10, 2025
eb3bdb6
[Subtask]: Add a registration function for table allocation in master…
Nov 10, 2025
caae047
[Subtask]: Replace zk with mocking. #3919
Nov 10, 2025
3c295b6
[Subtask]: Replace zk with mocking. #3919
Nov 10, 2025
9f06e80
[Subtask]: add AmsAssignService to implement balanced bucket allocati…
Nov 10, 2025
5a023cf
[Subtask]: add AmsAssignService to implement balanced bucket allocati…
Nov 11, 2025
127f682
[Subtask]: add AmsAssignService to implement balanced bucket allocati…
Nov 12, 2025
ee02411
[Subtask]: Modify DefaultTableService to be compatible with master-sl…
Nov 12, 2025
b9a57b9
[Subtask]: Modify the optimizer to support obtaining tasks from each …
Nov 17, 2025
ce6c004
[Subtask]: Supports forwarding OpenAPI requests to the master node in…
Nov 25, 2025
24cdfc1
[Subtask]: Fix unit test case failures. #3963
Nov 25, 2025
b4bc98c
This addresses the conflict issues with the latest main branch and in…
Dec 29, 2025
dc8a0dd
This addresses the conflict issues with the latest main branch and in…
Dec 29, 2025
e089b38
Troubleshooting unit test failures.
Dec 29, 2025
78f59dd
[Subtask]: Resolving conflicts with the latest master branch. #3963
Dec 30, 2025
fd4ef2e
[Subtask]: Supports forwarding OpenAPI requests to the master node in…
Apr 6, 2026
c132a76
[Subtask]: Supports forwarding OpenAPI requests to the master node in…
Apr 6, 2026
eaf0d76
[Subtask]: Supports forwarding OpenAPI requests to the master node in…
Apr 6, 2026
c514ce0
[Subtask]: Supports forwarding OpenAPI requests to the master node in…
Apr 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,50 @@ public class AmoroManagementConf {
.withDescription(
"Interval for syncing tables assigned to bucket IDs in master-slave mode. Each node periodically loads tables from database based on its assigned bucket IDs.");

public static final ConfigOption<Duration> HA_REQUEST_FORWARDER_TIMEOUT =
ConfigOptions.key("ha.request-forwarder.timeout")
.durationType()
.defaultValue(Duration.ofSeconds(30))
.withDescription("Timeout duration for request forwarding to leader node.");

public static final ConfigOption<Integer> HA_REQUEST_FORWARDER_MAX_RETRIES =
ConfigOptions.key("ha.request-forwarder.max-retries")
.intType()
.defaultValue(3)
.withDescription("Maximum number of retry attempts for request forwarding.");

public static final ConfigOption<Duration> HA_REQUEST_FORWARDER_RETRY_BACKOFF =
ConfigOptions.key("ha.request-forwarder.retry-backoff")
.durationType()
.defaultValue(Duration.ofSeconds(1))
.withDescription("Backoff duration between retry attempts for request forwarding.");

public static final ConfigOption<Integer> HA_REQUEST_FORWARDER_CIRCUIT_BREAKER_THRESHOLD =
ConfigOptions.key("ha.request-forwarder.circuit-breaker.threshold")
.intType()
.defaultValue(5)
.withDescription("Number of consecutive failures before opening the circuit breaker.");

public static final ConfigOption<Duration> HA_REQUEST_FORWARDER_CIRCUIT_BREAKER_TIMEOUT =
ConfigOptions.key("ha.request-forwarder.circuit-breaker.timeout")
.durationType()
.defaultValue(Duration.ofMinutes(1))
.withDescription(
"Timeout duration for circuit breaker to remain open before attempting to close.");

public static final ConfigOption<Integer> HA_REQUEST_FORWARDER_MAX_CONNECTIONS =
ConfigOptions.key("ha.request-forwarder.max-connections")
.intType()
.defaultValue(100)
.withDescription("Maximum number of connections for request forwarding HTTP client.");

public static final ConfigOption<Integer> HA_REQUEST_FORWARDER_MAX_CONNECTIONS_PER_ROUTE =
ConfigOptions.key("ha.request-forwarder.max-connections-per-route")
.intType()
.defaultValue(20)
.withDescription(
"Maximum number of connections per route for request forwarding HTTP client.");

public static final ConfigOption<Integer> TABLE_SERVICE_THRIFT_BIND_PORT =
ConfigOptions.key("thrift-server.table-service.bind-port")
.intType()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,15 @@
import org.apache.amoro.config.Configurations;
import org.apache.amoro.config.shade.utils.ConfigShadeUtils;
import org.apache.amoro.exception.AmoroRuntimeException;
import org.apache.amoro.exception.RequestForwardedException;
import org.apache.amoro.process.ActionCoordinator;
import org.apache.amoro.process.ExecuteEngine;
import org.apache.amoro.process.ProcessFactory;
import org.apache.amoro.server.catalog.CatalogManager;
import org.apache.amoro.server.catalog.DefaultCatalogManager;
import org.apache.amoro.server.dashboard.DashboardServer;
import org.apache.amoro.server.dashboard.JavalinJsonMapper;
import org.apache.amoro.server.dashboard.RequestForwarder;
import org.apache.amoro.server.dashboard.response.ErrorResponse;
import org.apache.amoro.server.dashboard.utils.AmsUtil;
import org.apache.amoro.server.dashboard.utils.CommonUtil;
Expand Down Expand Up @@ -128,6 +130,7 @@ public class AmoroServiceContainer {
private AmsServiceMetrics amsServiceMetrics;
private HAState haState = HAState.INITIALIZING;
private AmsAssignService amsAssignService;
private RequestForwarder requestForwarder;

public AmoroServiceContainer() throws Exception {
initConfig();
Expand Down Expand Up @@ -360,6 +363,11 @@ public void disposeRestService() {
if (amsServiceMetrics != null) {
amsServiceMetrics.unregister();
}
if (requestForwarder != null) {
LOG.info("Closing request forwarder...");
requestForwarder.close();
requestForwarder = null;
}

EventsManager.dispose();
MetricManager.dispose();
Expand Down Expand Up @@ -393,13 +401,69 @@ private void startThriftServer(TServer server, String threadName) {
}

private void initHttpService() {
// Create request forwarder for master-slave mode
requestForwarder = null;
if (haContainer != null && IS_MASTER_SLAVE_MODE) {
// Get configuration values for request forwarder
int timeoutMs =
(int) serviceConfig.get(AmoroManagementConf.HA_REQUEST_FORWARDER_TIMEOUT).toMillis();
int maxRetries =
serviceConfig.getInteger(AmoroManagementConf.HA_REQUEST_FORWARDER_MAX_RETRIES);
int retryBackoffMs =
(int)
serviceConfig.get(AmoroManagementConf.HA_REQUEST_FORWARDER_RETRY_BACKOFF).toMillis();
int circuitBreakerThreshold =
serviceConfig.getInteger(
AmoroManagementConf.HA_REQUEST_FORWARDER_CIRCUIT_BREAKER_THRESHOLD);
long circuitBreakerTimeoutMs =
serviceConfig
.get(AmoroManagementConf.HA_REQUEST_FORWARDER_CIRCUIT_BREAKER_TIMEOUT)
.toMillis();
int maxConnections =
serviceConfig.getInteger(AmoroManagementConf.HA_REQUEST_FORWARDER_MAX_CONNECTIONS);
int maxConnectionsPerRoute =
serviceConfig.getInteger(
AmoroManagementConf.HA_REQUEST_FORWARDER_MAX_CONNECTIONS_PER_ROUTE);

requestForwarder =
new org.apache.amoro.server.dashboard.RequestForwarder(
haContainer,
timeoutMs,
maxRetries,
retryBackoffMs,
circuitBreakerThreshold,
circuitBreakerTimeoutMs,
maxConnections,
maxConnectionsPerRoute,
IS_MASTER_SLAVE_MODE);

LOG.info(
"Request forwarder initialized with configuration: timeout={}ms, maxRetries={}, "
+ "retryBackoff={}ms, circuitBreakerThreshold={}, circuitBreakerTimeout={}ms, "
+ "maxConnections={}, maxConnectionsPerRoute={}",
timeoutMs,
maxRetries,
retryBackoffMs,
circuitBreakerThreshold,
circuitBreakerTimeoutMs,
maxConnections,
maxConnectionsPerRoute);
}

DashboardServer dashboardServer =
new DashboardServer(
serviceConfig, catalogManager, tableManager, optimizerManager, terminalManager, this);
serviceConfig,
catalogManager,
tableManager,
optimizerManager,
terminalManager,
this,
requestForwarder);
RestExtensionManager restExtensionManager = new RestExtensionManager();
restExtensionManager.initialize();
List<RestExtension> restExtensions =
restExtensionManager.loadExtensions(serviceConfig, catalogManager, tableManager);
restExtensionManager.loadExtensions(
serviceConfig, catalogManager, tableManager, requestForwarder);
Function<Context, Optional<RestExtension>> handleExceptionByExtension =
ctx -> restExtensions.stream().filter(ext -> ext.needHandleException(ctx)).findFirst();

Expand Down Expand Up @@ -433,6 +497,79 @@ private void initHttpService() {
dashboardServer.preHandleRequest(ctx);
}
});

// Handle RequestForwardedException - request was successfully forwarded to leader
// This must be registered before the generic Exception handler
httpServer.exception(
RequestForwardedException.class,
(e, ctx) -> {
// Request was forwarded, response data is stored in the exception
// Re-apply response data to ensure it's not lost during exception handling
if (e.hasResponseData()) {
// Set status code
ctx.status(e.getStatusCode());

// Set response body
byte[] responseBody = e.getResponseBody();
if (responseBody != null) {
// For 204/304, don't set body
if (e.getStatusCode() != 204 && e.getStatusCode() != 304) {
ctx.result(responseBody);
}
} else if (e.getStatusCode() != 204
&& e.getStatusCode() != 304
&& ctx.path().startsWith("/api/")) {
// Fallback: set empty JSON if body is null for API endpoints
ctx.result("{}".getBytes(java.nio.charset.StandardCharsets.UTF_8));
}

// Set content type
String contentType = e.getContentType();
if (contentType != null && !contentType.isEmpty()) {
ctx.contentType(contentType);
} else if (e.getStatusCode() != 204
&& e.getStatusCode() != 304
&& ctx.path().startsWith("/api/")) {
ctx.contentType("application/json");
}

// Set response headers
if (e.getResponseHeaders() != null) {
for (java.util.Map.Entry<String, String> entry : e.getResponseHeaders().entrySet()) {
ctx.header(entry.getKey(), entry.getValue());
}
}

LOG.debug(
"RequestForwardedException handled: restored response status {}, body-size: {}, content-type: {} for path: {}",
e.getStatusCode(),
responseBody != null ? responseBody.length : 0,
contentType,
ctx.path());
} else {
// Response data not available in exception, check if it was set in context
int currentStatus = ctx.status();
if (currentStatus > 0) {
LOG.debug(
"RequestForwardedException caught: response already set in context, status: {} for path: {}",
currentStatus,
ctx.path());
} else {
LOG.error(
"RequestForwardedException caught but no response data available! Path: {}",
ctx.path());
// Set a proper error response
ctx.status(io.javalin.http.HttpCode.INTERNAL_SERVER_ERROR);
ctx.contentType("application/json");
ctx.json(
new org.apache.amoro.server.dashboard.response.ErrorResponse(
io.javalin.http.HttpCode.INTERNAL_SERVER_ERROR,
"Request forwarding completed but response was not properly set",
""));
}
}
});

httpServer.exception(
Exception.class,
(e, ctx) -> {
Expand Down Expand Up @@ -667,6 +804,11 @@ private void initContainerConfig() {
containerProperties.putIfAbsent(
OptimizerProperties.AMS_OPTIMIZER_URI,
AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_OPTIMIZING_SERVICE_NAME));
// Add master-slave mode flag to container properties
if (serviceConfig.getBoolean(USE_MASTER_SLAVE_MODE)) {
containerProperties.put(
OptimizerProperties.OPTIMIZER_MASTER_SLAVE_MODE_ENABLED, "true");
}
// put addition system properties
container.setProperties(containerProperties);
containerList.add(container);
Expand Down
Loading
Loading