diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 078ac5997477..7ac60371db2e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2669,5 +2669,5 @@ List getLogEntries(Set serverNames, String logType, Server * Refresh the system key cache on all specified region servers. * @param regionServers the list of region servers to refresh the system key cache on */ - void refreshSystemKeyCacheOnAllServers(Set regionServers) throws IOException; + void refreshSystemKeyCacheOnServers(Set regionServers) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java index 5ae99b00a7e0..89233cabedca 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java @@ -1148,7 +1148,7 @@ public void restoreBackupSystemTable(String snapshotName) throws IOException { } @Override - public void refreshSystemKeyCacheOnAllServers(Set regionServers) throws IOException { - get(admin.refreshSystemKeyCacheOnAllServers(regionServers)); + public void refreshSystemKeyCacheOnServers(Set regionServers) throws IOException { + get(admin.refreshSystemKeyCacheOnServers(regionServers)); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index a10746f2726b..dc2bb37153ef 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1879,5 +1879,5 @@ CompletableFuture> getLogEntries(Set serverNames, Str * Refresh the system key cache on all specified region servers. * @param regionServers the list of region servers to refresh the system key cache on */ - CompletableFuture refreshSystemKeyCacheOnAllServers(Set regionServers); + CompletableFuture refreshSystemKeyCacheOnServers(Set regionServers); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index d135063ec9a2..ac509c20be91 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -687,8 +687,8 @@ public CompletableFuture updateConfiguration(String groupName) { } @Override - public CompletableFuture refreshSystemKeyCacheOnAllServers(Set regionServers) { - return wrap(rawAdmin.refreshSystemKeyCacheOnAllServers(regionServers)); + public CompletableFuture refreshSystemKeyCacheOnServers(Set regionServers) { + return wrap(rawAdmin.refreshSystemKeyCacheOnServers(regionServers)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 8a5033ff9b18..0af2625bb3b1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -4665,7 +4665,7 @@ MasterProtos.RestoreBackupSystemTableResponse> procedureCall(request, } @Override - public CompletableFuture refreshSystemKeyCacheOnAllServers(Set regionServers) { + public CompletableFuture refreshSystemKeyCacheOnServers(Set regionServers) { CompletableFuture future = new CompletableFuture<>(); List> futures = regionServers.stream().map(this::refreshSystemKeyCache).collect(Collectors.toList()); diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java index 9b4d013e7b65..bf550b3a0472 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java @@ -127,6 +127,7 @@ public enum LockState { private long rootProcId = NO_PROC_ID; private long procId = NO_PROC_ID; private long submittedTime; + private boolean isCriticalSystemTable; // Runtime state, updated every operation private ProcedureState state = ProcedureState.INITIALIZING; @@ -608,6 +609,14 @@ protected void setParentProcId(long parentProcId) { this.parentProcId = parentProcId; } + public void setCriticalSystemTable(boolean isCriticalSystemTable) { + this.isCriticalSystemTable = isCriticalSystemTable; + } + + public boolean isCriticalSystemTable() { + return isCriticalSystemTable; + } + protected void setRootProcId(long rootProcId) { this.rootProcId = rootProcId; } diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 1f0029e98744..9f8c4197281b 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -1975,6 +1975,7 @@ private Procedure[] initializeChildren(RootProcedureState pro builder.setParentId(proc.getParentProcId()); } + if (proc.isCriticalSystemTable()) { + builder.setIsCryticalSystemTable(true); + } + if (proc.hasTimeout()) { builder.setTimeout(proc.getTimeout()); } @@ -244,6 +248,10 @@ public static Procedure convertToProcedure(ProcedureProtos.Procedure proto) proc.setParentProcId(proto.getParentId()); } + if (proto.hasIsCryticalSystemTable()) { + proc.setCriticalSystemTable(proto.getIsCryticalSystemTable()); + } + if (proto.hasOwner()) { proc.setOwner(proto.getOwner()); } diff --git a/hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto index f75e858549a0..0ccf44f8e283 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto @@ -73,6 +73,9 @@ message Procedure { // whether the procedure has been executed // since we do not always maintain the stack_id now, we need a separated flag optional bool executed = 18 [default = false]; + + // Indicates that the procedure belongs to a crytical system table. + optional bool is_crytical_system_table = 19 [default = false]; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java index 030a278f023e..894606e9a2b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java @@ -94,7 +94,7 @@ public boolean rotateSTK() throws IOException { LOG.info("System Key is rotated, initiating cache refresh on all region servers"); try { - FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnAllServers(regionServers)); + FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnServers(regionServers)); } catch (Exception e) { throw new IOException( "Failed to initiate System Key cache refresh on one or more region servers", e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaMasterService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaMasterService.java deleted file mode 100644 index c33a331ba04a..000000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaMasterService.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.keymeta; - -import java.io.IOException; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@InterfaceAudience.Private -public class KeymetaMasterService extends KeyManagementBase { - private static final Logger LOG = LoggerFactory.getLogger(KeymetaMasterService.class); - - private final MasterServices master; - - private static final TableDescriptorBuilder TABLE_DESCRIPTOR_BUILDER = - TableDescriptorBuilder.newBuilder(KeymetaTableAccessor.KEY_META_TABLE_NAME) - .setRegionReplication(1).setPriority(HConstants.SYSTEMTABLE_QOS) - .setColumnFamily(ColumnFamilyDescriptorBuilder - .newBuilder(KeymetaTableAccessor.KEY_META_INFO_FAMILY) - .setScope(HConstants.REPLICATION_SCOPE_LOCAL).setMaxVersions(1).setInMemory(true).build()); - - public KeymetaMasterService(MasterServices masterServices) { - super(masterServices); - master = masterServices; - } - - public void init() throws IOException { - if (!isKeyManagementEnabled()) { - return; - } - if (!master.getTableDescriptors().exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)) { - LOG.info("{} table not found. Creating.", - KeymetaTableAccessor.KEY_META_TABLE_NAME.getNameWithNamespaceInclAsString()); - master.createSystemTable(TABLE_DESCRIPTOR_BUILDER.build()); - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaTableAccessor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaTableAccessor.java index 8e2a7095cfca..e6a7d288fddb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaTableAccessor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaTableAccessor.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; @@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.io.crypto.ManagedKeyData; import org.apache.hadoop.hbase.io.crypto.ManagedKeyState; @@ -76,6 +78,13 @@ public class KeymetaTableAccessor extends KeyManagementBase { public static final String KEY_STATE_QUAL_NAME = "k"; public static final byte[] KEY_STATE_QUAL_BYTES = Bytes.toBytes(KEY_STATE_QUAL_NAME); + public static final TableDescriptorBuilder TABLE_DESCRIPTOR_BUILDER = + TableDescriptorBuilder.newBuilder(KEY_META_TABLE_NAME).setRegionReplication(1) + .setPriority(HConstants.SYSTEMTABLE_QOS) + .setColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder(KeymetaTableAccessor.KEY_META_INFO_FAMILY) + .setScope(HConstants.REPLICATION_SCOPE_LOCAL).setMaxVersions(1).setInMemory(true).build()); + private Server server; public KeymetaTableAccessor(Server server) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/ManagedKeyDataCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/ManagedKeyDataCache.java index 3f3c66a8bccd..76a6bdb8f915 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/ManagedKeyDataCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/ManagedKeyDataCache.java @@ -247,7 +247,7 @@ public ManagedKeyData getActiveEntry(byte[] key_cust, String keyNamespace) { }); // This should never be null, but adding a check just to satisfy spotbugs. - if (keyData!= null && keyData.getKeyState() == ManagedKeyState.ACTIVE) { + if (keyData != null && keyData.getKeyState() == ManagedKeyState.ACTIVE) { return keyData; } return null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 7037656419b3..530fd5c41951 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -123,7 +123,7 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -import org.apache.hadoop.hbase.keymeta.KeymetaMasterService; +import org.apache.hadoop.hbase.keymeta.KeymetaTableAccessor; import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode; import org.apache.hadoop.hbase.master.assignment.AssignmentManager; @@ -248,6 +248,7 @@ import org.apache.hadoop.hbase.rsgroup.RSGroupUtil; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.SecurityConstants; +import org.apache.hadoop.hbase.security.SecurityUtil; import org.apache.hadoop.hbase.security.Superusers; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.trace.TraceUtil; @@ -360,7 +361,6 @@ public class HMaster extends HBaseServerBase implements Maste private MasterFileSystem fileSystemManager; private MasterWalManager walManager; private SystemKeyManager systemKeyManager; - private KeymetaMasterService keymetaMasterService; // manager to manage procedure-based WAL splitting, can be null if current // is zk-based WAL splitting. SplitWALManager will replace SplitLogManager @@ -1041,9 +1041,6 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE Map, List>> procsByType = procedureExecutor .getActiveProceduresNoCopy().stream().collect(Collectors.groupingBy(p -> p.getClass())); - keymetaMasterService = new KeymetaMasterService(this); - keymetaMasterService.init(); - // Create Assignment Manager this.assignmentManager = createAssignmentManager(this, masterRegion); this.assignmentManager.start(); @@ -1166,13 +1163,22 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE return; } + this.assignmentManager.joinCluster(); + + // If key management is enabled, wait for keymeta table regions to be assigned and online, + // which includes creating the table the very first time. + // This is to ensure that the encrypted tables can successfully initialize Encryption.Context as + // part of the store opening process when processOfflineRegions is called. + // Without this, we can end up with race condition where a user store is opened before the + // keymeta table regions are online, which would cause the store opening to fail. + initKeymetaIfEnabled(); + TableDescriptor metaDescriptor = tableDescriptors.get(TableName.META_TABLE_NAME); final ColumnFamilyDescriptor tableFamilyDesc = metaDescriptor.getColumnFamily(HConstants.TABLE_FAMILY); final ColumnFamilyDescriptor replBarrierFamilyDesc = metaDescriptor.getColumnFamily(HConstants.REPLICATION_BARRIER_FAMILY); - this.assignmentManager.joinCluster(); // The below depends on hbase:meta being online. this.assignmentManager.processOfflineRegions(); // this must be called after the above processOfflineRegions to prevent race @@ -1433,6 +1439,10 @@ private void createMissingCFsInMetaDuringUpgrade(TableDescriptor metaDescriptor) .setColumnFamily(FSTableDescriptors.getTableFamilyDescForMeta(conf)) .setColumnFamily(FSTableDescriptors.getReplBarrierFamilyDescForMeta()).build(); long pid = this.modifyTable(TableName.META_TABLE_NAME, () -> newMetaDesc, 0, 0, false); + waitForProcedureToComplete(pid, "Failed to add table and rep_barrier CFs to meta"); + } + + private void waitForProcedureToComplete(long pid, String errorMessage) throws IOException { int tries = 30; while ( !(getMasterProcedureExecutor().isFinished(pid)) && getMasterProcedureExecutor().isRunning() @@ -1451,8 +1461,8 @@ private void createMissingCFsInMetaDuringUpgrade(TableDescriptor metaDescriptor) } else { Procedure result = getMasterProcedureExecutor().getResult(pid); if (result != null && result.isFailed()) { - throw new IOException("Failed to add table and rep_barrier CFs to meta. " - + MasterProcedureUtil.unwrapRemoteIOException(result)); + throw new IOException( + errorMessage + ". " + MasterProcedureUtil.unwrapRemoteIOException(result)); } } } @@ -1530,6 +1540,80 @@ private boolean waitForNamespaceOnline() throws IOException { return true; } + /** + * Creates the keymeta table and waits for all its regions to be online. + */ + private void initKeymetaIfEnabled() throws IOException { + if (!SecurityUtil.isKeyManagementEnabled(conf)) { + return; + } + + String keymetaTableName = + KeymetaTableAccessor.KEY_META_TABLE_NAME.getNameWithNamespaceInclAsString(); + if (!getTableDescriptors().exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)) { + LOG.info("initKeymetaIfEnabled: {} table not found. Creating.", keymetaTableName); + long keymetaTableProcId = + createSystemTable(KeymetaTableAccessor.TABLE_DESCRIPTOR_BUILDER.build(), true); + + LOG.info("initKeymetaIfEnabled: Waiting for {} table creation procedure {} to complete", + keymetaTableName, keymetaTableProcId); + waitForProcedureToComplete(keymetaTableProcId, + "Failed to create keymeta table and add to meta"); + } + + List ris = this.assignmentManager.getRegionStates() + .getRegionsOfTable(KeymetaTableAccessor.KEY_META_TABLE_NAME); + if (ris.isEmpty()) { + throw new RuntimeException( + "initKeymetaIfEnabled: No " + keymetaTableName + " table regions found"); + } + + // First, create assignment procedures for all keymeta regions + List procs = new ArrayList<>(); + for (RegionInfo ri : ris) { + RegionStateNode regionNode = + assignmentManager.getRegionStates().getOrCreateRegionStateNode(ri); + regionNode.lock(); + try { + // Only create if region is in CLOSED or OFFLINE state and no procedure is already attached. + // The check for server online is really needed only for the sake of mini cluster as there + // are outdated ONLINE entries being returned after a cluster restart that point to the old + // RS. + if ( + (regionNode.isInState(RegionState.State.CLOSED, RegionState.State.OFFLINE) + || !this.serverManager.isServerOnline(regionNode.getRegionLocation())) + && regionNode.getProcedure() == null + ) { + TransitRegionStateProcedure proc = TransitRegionStateProcedure + .assign(getMasterProcedureExecutor().getEnvironment(), ri, null); + proc.setCriticalSystemTable(true); + regionNode.setProcedure(proc); + procs.add(proc); + } + } finally { + regionNode.unlock(); + } + } + // Then, trigger assignment for all keymeta regions + if (!procs.isEmpty()) { + LOG.info("initKeymetaIfEnabled: Submitting {} assignment procedures for {} table regions", + procs.size(), keymetaTableName); + getMasterProcedureExecutor() + .submitProcedures(procs.toArray(new TransitRegionStateProcedure[procs.size()])); + } + + // Then wait for all regions to come online + LOG.info("initKeymetaIfEnabled: Checking/Waiting for {} table {} regions to be online", + keymetaTableName, ris.size()); + for (RegionInfo ri : ris) { + if (!isRegionOnline(ri)) { + throw new RuntimeException(keymetaTableName + " table region " + ri.getRegionNameAsString() + + " could not be brought online"); + } + } + LOG.info("initKeymetaIfEnabled: All {} table regions are online", keymetaTableName); + } + /** * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to * automatically remove quotas for a table when that table is deleted. @@ -2528,6 +2612,11 @@ protected String getDescription() { @Override public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException { + return createSystemTable(tableDescriptor, false); + } + + private long createSystemTable(final TableDescriptor tableDescriptor, final boolean isCritical) + throws IOException { if (isStopped()) { throw new MasterNotRunningException(); } @@ -2544,10 +2633,10 @@ public long createSystemTable(final TableDescriptor tableDescriptor) throws IOEx // This special create table is called locally to master. Therefore, no RPC means no need // to use nonce to detect duplicated RPC call. - long procId = this.procedureExecutor.submitProcedure( - new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions)); - - return procId; + CreateTableProcedure proc = + new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions); + proc.setCriticalSystemTable(isCritical); + return this.procedureExecutor.submitProcedure(proc); } private void startActiveMasterManager(int infoPort) throws KeeperException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 8c37385757a7..a964dd3b7046 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1285,7 +1285,7 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( private void updateRegionTransition(ServerStateNode serverNode, TransitionCode state, RegionInfo regionInfo, long seqId, long procId) throws IOException { - checkMetaLoaded(regionInfo); + checkMetaLoaded(regionInfo, procId); RegionStateNode regionNode = regionStates.getRegionStateNode(regionInfo); if (regionNode == null) { @@ -1337,7 +1337,7 @@ private boolean reportTransition(RegionStateNode regionNode, ServerStateNode ser private void updateRegionSplitTransition(final ServerStateNode serverNode, final TransitionCode state, final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB) throws IOException { - checkMetaLoaded(parent); + checkMetaLoaded(parent, Procedure.NO_PROC_ID); if (state != TransitionCode.READY_TO_SPLIT) { throw new UnexpectedStateException( @@ -1391,7 +1391,7 @@ private void updateRegionSplitTransition(final ServerStateNode serverNode, private void updateRegionMergeTransition(final ServerStateNode serverNode, final TransitionCode state, final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException { - checkMetaLoaded(merged); + checkMetaLoaded(merged, Procedure.NO_PROC_ID); if (state != TransitionCode.READY_TO_MERGE) { throw new UnexpectedStateException( @@ -1911,13 +1911,27 @@ private void loadMeta() throws IOException { * Used to check if the meta loading is done. *

* if not we throw PleaseHoldException since we are rebuilding the RegionStates - * @param hri region to check if it is already rebuild + * @param hri region to check if it is already rebuild + * @param procId the procedure id for this region operation, or NO_PROC_ID if not available * @throws PleaseHoldException if meta has not been loaded yet */ - private void checkMetaLoaded(RegionInfo hri) throws PleaseHoldException { + private void checkMetaLoaded(RegionInfo hri, long procId) throws PleaseHoldException { if (!isRunning()) { throw new PleaseHoldException("AssignmentManager not running"); } + + // Check if the procedure is for a critical system table + // Critical system tables can proceed even if meta is not loaded yet + // We are currently making procId available only for the code path which can execute during the + // cluster boot up. In the future, if additional code paths execute during cluster boot up, we + // will need to make procId available for all those code paths. + if (procId != Procedure.NO_PROC_ID) { + Procedure proc = master.getMasterProcedureExecutor().getProcedure(procId); + if (proc != null && proc.isCriticalSystemTable()) { + return; + } + } + boolean meta = isMetaRegion(hri); boolean metaLoaded = isMetaLoaded(); if (!meta && !metaLoaded) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java index cb3b91ca0e20..fe8c9bfdd7c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java @@ -155,6 +155,9 @@ public TableName getTableName() { @Override protected boolean waitInitialized(MasterProcedureEnv env) { + if (isCriticalSystemTable()) { + return false; + } if (TableName.isMetaTableName(getTableName())) { return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java index 12ddb2559367..5f02fea9c463 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.java @@ -206,6 +206,9 @@ public TableOperationType getTableOperationType() { @Override protected boolean waitInitialized(MasterProcedureEnv env) { + if (isCriticalSystemTable()) { + return false; + } if (TableName.isMetaTableName(getTableName())) { return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 59a8285b2f65..6f7ab6b82104 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -264,6 +264,9 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws @Override protected boolean waitInitialized(MasterProcedureEnv env) { + if (isCriticalSystemTable()) { + return false; + } if (getTableName().isSystemTable()) { // Creating system table is part of the initialization, so only wait for meta loaded instead // of waiting for master fully initialized. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java index 3c337ce72131..9f2381e849bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/ManagedKeyTestBase.java @@ -35,6 +35,8 @@ public class ManagedKeyTestBase { @Before public void setUp() throws Exception { + // Uncomment to enable trace logging for the tests that extend this base class. + // Log4jUtils.setLogLevel("org.apache.hadoop.hbase", "TRACE"); if (isWithKeyManagement()) { TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MANAGED_KEYPROVIDER_CONF_KEY, getKeyProviderClass().getName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaMasterService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaMasterService.java deleted file mode 100644 index 9ccb3dc2568f..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/keymeta/TestKeymetaMasterService.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.keymeta; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableDescriptors; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -/** - * Tests for KeymetaMasterService class - */ -@Category({ MasterTests.class, SmallTests.class }) -public class TestKeymetaMasterService { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestKeymetaMasterService.class); - - @Mock - private MasterServices mockMaster; - @Mock - private TableDescriptors mockTableDescriptors; - - private Configuration conf; - private KeymetaMasterService service; - private AutoCloseable closeableMocks; - - @Before - public void setUp() throws Exception { - closeableMocks = MockitoAnnotations.openMocks(this); - - conf = new Configuration(); - when(mockMaster.getConfiguration()).thenReturn(conf); - when(mockMaster.getTableDescriptors()).thenReturn(mockTableDescriptors); - } - - @After - public void tearDown() throws Exception { - if (closeableMocks != null) { - closeableMocks.close(); - } - } - - @Test - public void testInitWithKeyManagementDisabled() throws Exception { - // Setup - disable key management - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, false); - - service = new KeymetaMasterService(mockMaster); - - // Execute - service.init(); // Should return early without creating table - - // Verify - no table operations should be performed - verify(mockMaster, never()).getTableDescriptors(); - verify(mockMaster, never()).createSystemTable(any()); - } - - @Test - public void testInitWithKeyManagementEnabledAndTableExists() throws Exception { - // Setup - enable key management and table already exists - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockTableDescriptors.exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)).thenReturn(true); - - service = new KeymetaMasterService(mockMaster); - - // Execute - service.init(); - - // Verify - table exists check is performed but no table creation - verify(mockMaster).getTableDescriptors(); - verify(mockTableDescriptors).exists(KeymetaTableAccessor.KEY_META_TABLE_NAME); - verify(mockMaster, never()).createSystemTable(any()); - } - - @Test - public void testInitWithKeyManagementEnabledAndTableDoesNotExist() throws Exception { - // Setup - enable key management and table does not exist - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockTableDescriptors.exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)).thenReturn(false); - - service = new KeymetaMasterService(mockMaster); - - // Execute - service.init(); - - // Verify - table is created - verify(mockMaster).getTableDescriptors(); - verify(mockTableDescriptors).exists(KeymetaTableAccessor.KEY_META_TABLE_NAME); - verify(mockMaster).createSystemTable(any(TableDescriptor.class)); - } - - @Test - public void testInitWithTableDescriptorsIOException() throws Exception { - // Setup - enable key management but table descriptors throws IOException - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockTableDescriptors.exists(any(TableName.class))) - .thenThrow(new IOException("Table descriptors error")); - - service = new KeymetaMasterService(mockMaster); - - // Execute & Verify - IOException should propagate - try { - service.init(); - } catch (IOException e) { - // Expected exception - } - - verify(mockMaster).getTableDescriptors(); - verify(mockTableDescriptors).exists(KeymetaTableAccessor.KEY_META_TABLE_NAME); - verify(mockMaster, never()).createSystemTable(any()); - } - - @Test - public void testInitWithCreateSystemTableIOException() throws Exception { - // Setup - enable key management, table doesn't exist, but createSystemTable throws IOException - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockTableDescriptors.exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)).thenReturn(false); - when(mockMaster.createSystemTable(any(TableDescriptor.class))) - .thenThrow(new IOException("Create table error")); - - service = new KeymetaMasterService(mockMaster); - - // Execute & Verify - IOException should propagate - try { - service.init(); - } catch (IOException e) { - // Expected exception - } - - verify(mockMaster).getTableDescriptors(); - verify(mockTableDescriptors).exists(KeymetaTableAccessor.KEY_META_TABLE_NAME); - verify(mockMaster).createSystemTable(any(TableDescriptor.class)); - } - - @Test - public void testConstructorWithMasterServices() throws Exception { - // Execute - service = new KeymetaMasterService(mockMaster); - - // Verify - constructor should not throw an exception - // The service should be created successfully (no exceptions = success) - // We don't verify internal calls since the constructor just stores references - } - - @Test - public void testMultipleInitCalls() throws Exception { - // Setup - enable key management and table exists - conf.setBoolean(HConstants.CRYPTO_MANAGED_KEYS_ENABLED_CONF_KEY, true); - when(mockTableDescriptors.exists(KeymetaTableAccessor.KEY_META_TABLE_NAME)).thenReturn(true); - - service = new KeymetaMasterService(mockMaster); - - // Execute - call init multiple times - service.init(); - service.init(); - service.init(); - - // Verify - each call should check table existence (idempotent behavior) - verify(mockMaster, times(3)).getTableDescriptors(); - verify(mockTableDescriptors, times(3)).exists(KeymetaTableAccessor.KEY_META_TABLE_NAME); - verify(mockMaster, never()).createSystemTable(any()); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java index cb5c8dc11747..9500a54b31e8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestKeymetaAdminImpl.java @@ -308,7 +308,7 @@ public void testRotateSTKWithNewKey() throws Exception { // Mock SystemKeyManager to return a new key (non-null) when(mockServer.rotateSystemKeyIfChanged()).thenReturn(true); - when(mockAsyncAdmin.refreshSystemKeyCacheOnAllServers(any())) + when(mockAsyncAdmin.refreshSystemKeyCacheOnServers(any())) .thenReturn(CompletableFuture.completedFuture(null)); KeymetaAdminImplForTest admin = new KeymetaAdminImplForTest(mockServer, keymetaAccessor); @@ -321,7 +321,7 @@ public void testRotateSTKWithNewKey() throws Exception { // Verify that rotateSystemKeyIfChanged was called verify(mockServer).rotateSystemKeyIfChanged(); - verify(mockAsyncAdmin).refreshSystemKeyCacheOnAllServers(any()); + verify(mockAsyncAdmin).refreshSystemKeyCacheOnServers(any()); } /** @@ -371,7 +371,7 @@ public void testRotateSTKWithFailedServerRefresh() throws Exception { CompletableFuture failedFuture = new CompletableFuture<>(); failedFuture.completeExceptionally(new IOException("refresh failed")); - when(mockAsyncAdmin.refreshSystemKeyCacheOnAllServers(any())).thenReturn(failedFuture); + when(mockAsyncAdmin.refreshSystemKeyCacheOnServers(any())).thenReturn(failedFuture); KeymetaAdminImplForTest admin = new KeymetaAdminImplForTest(mockServer, keymetaAccessor); @@ -383,7 +383,7 @@ public void testRotateSTKWithFailedServerRefresh() throws Exception { // Verify that rotateSystemKeyIfChanged was called verify(mockServer).rotateSystemKeyIfChanged(); - verify(mockAsyncAdmin).refreshSystemKeyCacheOnAllServers(any()); + verify(mockAsyncAdmin).refreshSystemKeyCacheOnServers(any()); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 5e6b8db58243..71b24ecc954a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SingleProcessHBaseCluster; import org.apache.hadoop.hbase.StartTestingClusterOption; +import org.apache.hadoop.hbase.keymeta.ManagedKeyTestBase; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.FlakeyTests; @@ -40,10 +41,16 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.Suite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Category({ FlakeyTests.class, LargeTests.class }) +@RunWith(Suite.class) +@Suite.SuiteClasses({ TestMasterFailover.TestMasterFailoverDefaultConfig.class, + TestMasterFailover.TestSimpleMasterFailoverWithKeymeta.class }) public class TestMasterFailover { @ClassRule @@ -54,19 +61,11 @@ public class TestMasterFailover { @Rule public TestName name = new TestName(); - /** - * Simple test of master failover. - *

- * Starts with three masters. Kills a backup master. Then kills the active master. Ensures the - * final master becomes active and we can still contact the cluster. - */ - @Test - public void testSimpleMasterFailover() throws Exception { + protected static void doTestSimpleMasterFailover(HBaseTestingUtil TEST_UTIL) throws Exception { final int NUM_MASTERS = 3; final int NUM_RS = 3; // Start the cluster - HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); try { StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(NUM_MASTERS) .numRegionServers(NUM_RS).numDataNodes(NUM_RS).build(); @@ -168,50 +167,90 @@ public void testSimpleMasterFailover() throws Exception { } } - /** - * Test meta in transition when master failover. This test used to manipulate region state up in - * zk. That is not allowed any more in hbase2 so I removed that messing. That makes this test - * anemic. - */ - @Test - public void testMetaInTransitionWhenMasterFailover() throws Exception { - // Start the cluster - HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); - TEST_UTIL.startMiniCluster(); - try { - SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); - LOG.info("Cluster started"); - - HMaster activeMaster = cluster.getMaster(); - ServerName metaServerName = cluster.getServerHoldingMeta(); - HRegionServer hrs = cluster.getRegionServer(metaServerName); - - // Now kill master, meta should remain on rs, where we placed it before. - LOG.info("Aborting master"); - activeMaster.abort("test-kill"); - cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); - LOG.info("Master has aborted"); - - // meta should remain where it was - RegionState metaState = MetaTableLocator.getMetaRegionState(hrs.getZooKeeper()); - assertEquals("hbase:meta should be online on RS", metaState.getServerName(), metaServerName); - assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState()); - - // Start up a new master - LOG.info("Starting up a new master"); - activeMaster = cluster.startMaster().getMaster(); - LOG.info("Waiting for master to be ready"); - cluster.waitForActiveAndReadyMaster(); - LOG.info("Master is ready"); - - // ensure meta is still deployed on RS - metaState = MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper()); - assertEquals("hbase:meta should be online on RS", metaState.getServerName(), metaServerName); - assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState()); - - // Done, shutdown the cluster - } finally { - TEST_UTIL.shutdownMiniCluster(); + @RunWith(BlockJUnit4ClassRunner.class) + @Category({ FlakeyTests.class, LargeTests.class }) + public static class TestMasterFailoverDefaultConfig { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMasterFailoverDefaultConfig.class); + + /** + * Simple test of master failover. + *

+ * Starts with three masters. Kills a backup master. Then kills the active master. Ensures the + * final master becomes active and we can still contact the cluster. + */ + @Test + public void testSimpleMasterFailover() throws Exception { + HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + doTestSimpleMasterFailover(TEST_UTIL); + } + + /** + * Test meta in transition when master failover. This test used to manipulate region state up in + * zk. That is not allowed any more in hbase2 so I removed that messing. That makes this test + * anemic. + */ + @Test + public void testMetaInTransitionWhenMasterFailover() throws Exception { + // Start the cluster + HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + TEST_UTIL.startMiniCluster(); + try { + SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + LOG.info("Cluster started"); + + HMaster activeMaster = cluster.getMaster(); + ServerName metaServerName = cluster.getServerHoldingMeta(); + HRegionServer hrs = cluster.getRegionServer(metaServerName); + + // Now kill master, meta should remain on rs, where we placed it before. + LOG.info("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + LOG.info("Master has aborted"); + + // meta should remain where it was + RegionState metaState = MetaTableLocator.getMetaRegionState(hrs.getZooKeeper()); + assertEquals("hbase:meta should be online on RS", metaState.getServerName(), + metaServerName); + assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState()); + + // Start up a new master + LOG.info("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + LOG.info("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + LOG.info("Master is ready"); + + // ensure meta is still deployed on RS + metaState = MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be online on RS", metaState.getServerName(), + metaServerName); + assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState()); + + // Done, shutdown the cluster + } finally { + TEST_UTIL.shutdownMiniCluster(); + } + } + } + + @RunWith(BlockJUnit4ClassRunner.class) + @Category({ FlakeyTests.class, LargeTests.class }) + public static class TestSimpleMasterFailoverWithKeymeta extends ManagedKeyTestBase { + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSimpleMasterFailoverWithKeymeta.class); + + @Test + public void testSimpleMasterFailoverWithKeymeta() throws Exception { + doTestSimpleMasterFailover(TEST_UTIL); + } + + @Override + protected boolean isWithMiniClusterStart() { + return false; } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index 1ea386aba923..d65b0e3956c5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -1001,7 +1001,7 @@ public boolean isReplicationPeerModificationEnabled() throws IOException { } @Override - public void refreshSystemKeyCacheOnAllServers(Set regionServers) throws IOException { - admin.refreshSystemKeyCacheOnAllServers(regionServers); + public void refreshSystemKeyCacheOnServers(Set regionServers) throws IOException { + admin.refreshSystemKeyCacheOnServers(regionServers); } } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index daa67297e252..b26cf1ead4d5 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -1378,8 +1378,8 @@ public boolean isReplicationPeerModificationEnabled() throws IOException { } @Override - public void refreshSystemKeyCacheOnAllServers(Set regionServers) throws IOException { + public void refreshSystemKeyCacheOnServers(Set regionServers) throws IOException { throw new NotImplementedException( - "refreshSystemKeyCacheOnAllServers not supported in ThriftAdmin"); + "refreshSystemKeyCacheOnServers not supported in ThriftAdmin"); } }