Make the keymeta table initialization happen in a predictable manner

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -2669,5 +2669,5 @@ List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType, Server
    * Refresh the system key cache on all specified region servers.
    * @param regionServers the list of region servers to refresh the system key cache on
    */
-  void refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) throws IOException;
+  void refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) throws IOException;
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java

@@ -1148,7 +1148,7 @@ public void restoreBackupSystemTable(String snapshotName) throws IOException {
   }
 
   @Override
-  public void refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) throws IOException {
-    get(admin.refreshSystemKeyCacheOnAllServers(regionServers));
+  public void refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) throws IOException {
+    get(admin.refreshSystemKeyCacheOnServers(regionServers));
   }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java

@@ -1879,5 +1879,5 @@ CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNames, Str
    * Refresh the system key cache on all specified region servers.
    * @param regionServers the list of region servers to refresh the system key cache on
    */
-  CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers);
+  CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers);
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java

@@ -687,8 +687,8 @@ public CompletableFuture<Void> updateConfiguration(String groupName) {
   }
 
   @Override
-  public CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) {
-    return wrap(rawAdmin.refreshSystemKeyCacheOnAllServers(regionServers));
+  public CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) {
+    return wrap(rawAdmin.refreshSystemKeyCacheOnServers(regionServers));
   }
 
   @Override

hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java

@@ -4665,7 +4665,7 @@ MasterProtos.RestoreBackupSystemTableResponse> procedureCall(request,
   }
 
   @Override
-  public CompletableFuture<Void> refreshSystemKeyCacheOnAllServers(Set<ServerName> regionServers) {
+  public CompletableFuture<Void> refreshSystemKeyCacheOnServers(Set<ServerName> regionServers) {
     CompletableFuture<Void> future = new CompletableFuture<>();
     List<CompletableFuture<Void>> futures =
       regionServers.stream().map(this::refreshSystemKeyCache).collect(Collectors.toList());

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java

@@ -127,6 +127,7 @@ public enum LockState {
   private long rootProcId = NO_PROC_ID;
   private long procId = NO_PROC_ID;
   private long submittedTime;
+  private boolean isCriticalSystemTable;
 
   // Runtime state, updated every operation
   private ProcedureState state = ProcedureState.INITIALIZING;
@@ -608,6 +609,14 @@ protected void setParentProcId(long parentProcId) {
     this.parentProcId = parentProcId;
   }
 
+  public void setCriticalSystemTable(boolean isCriticalSystemTable) {
+    this.isCriticalSystemTable = isCriticalSystemTable;
+  }
+
+  public boolean isCriticalSystemTable() {
+    return isCriticalSystemTable;
+  }
+
   protected void setRootProcId(long rootProcId) {
     this.rootProcId = rootProcId;
   }

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java

@@ -1975,6 +1975,7 @@ private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnviron
       subproc.setParentProcId(procedure.getProcId());
       subproc.setRootProcId(rootProcId);
       subproc.setProcId(nextProcId());
+      subproc.setCriticalSystemTable(procedure.isCriticalSystemTable());
       procStack.addSubProcedure(subproc);
     }
 

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureUtil.java

@@ -175,6 +175,10 @@ public static ProcedureProtos.Procedure convertToProtoProcedure(Procedure<?> pro
       builder.setParentId(proc.getParentProcId());
     }
 
+    if (proc.isCriticalSystemTable()) {
+      builder.setIsCryticalSystemTable(true);
+    }
+
     if (proc.hasTimeout()) {
       builder.setTimeout(proc.getTimeout());
     }
@@ -244,6 +248,10 @@ public static Procedure<?> convertToProcedure(ProcedureProtos.Procedure proto)
       proc.setParentProcId(proto.getParentId());
     }
 
+    if (proto.hasIsCryticalSystemTable()) {
+      proc.setCriticalSystemTable(proto.getIsCryticalSystemTable());
+    }
+
     if (proto.hasOwner()) {
       proc.setOwner(proto.getOwner());
     }

hbase-protocol-shaded/src/main/protobuf/server/Procedure.proto

@@ -73,6 +73,9 @@ message Procedure {
   // whether the procedure has been executed
   // since we do not always maintain the stack_id now, we need a separated flag
   optional bool executed = 18 [default = false];
+
+  // Indicates that the procedure belongs to a crytical system table.
+  optional bool is_crytical_system_table = 19 [default = false];
 }
 
 /**

hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaAdminImpl.java

@@ -94,7 +94,7 @@ public boolean rotateSTK() throws IOException {
 
     LOG.info("System Key is rotated, initiating cache refresh on all region servers");
     try {
-      FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnAllServers(regionServers));
+      FutureUtils.get(getAsyncAdmin(master).refreshSystemKeyCacheOnServers(regionServers));
     } catch (Exception e) {
       throw new IOException(
         "Failed to initiate System Key cache refresh on one or more region servers", e);

hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/KeymetaTableAccessor.java

@@ -29,6 +29,7 @@
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Get;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.io.crypto.ManagedKeyData;
 import org.apache.hadoop.hbase.io.crypto.ManagedKeyState;
@@ -76,6 +78,13 @@ public class KeymetaTableAccessor extends KeyManagementBase {
   public static final String KEY_STATE_QUAL_NAME = "k";
   public static final byte[] KEY_STATE_QUAL_BYTES = Bytes.toBytes(KEY_STATE_QUAL_NAME);
 
+  public static final TableDescriptorBuilder TABLE_DESCRIPTOR_BUILDER =
+    TableDescriptorBuilder.newBuilder(KEY_META_TABLE_NAME).setRegionReplication(1)
+      .setPriority(HConstants.SYSTEMTABLE_QOS)
+      .setColumnFamily(ColumnFamilyDescriptorBuilder
+        .newBuilder(KeymetaTableAccessor.KEY_META_INFO_FAMILY)
+        .setScope(HConstants.REPLICATION_SCOPE_LOCAL).setMaxVersions(1).setInMemory(true).build());
+
   private Server server;
 
   public KeymetaTableAccessor(Server server) {

hbase-server/src/main/java/org/apache/hadoop/hbase/keymeta/ManagedKeyDataCache.java

@@ -247,7 +247,7 @@ public ManagedKeyData getActiveEntry(byte[] key_cust, String keyNamespace) {
     });
 
     // This should never be null, but adding a check just to satisfy spotbugs.
-    if (keyData!= null && keyData.getKeyState() == ManagedKeyState.ACTIVE) {
+    if (keyData != null && keyData.getKeyState() == ManagedKeyState.ACTIVE) {
       return keyData;
     }
     return null;