report dirty metadata in meta table

hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java

@@ -37,10 +37,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.*;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -92,6 +89,26 @@ public Map<TableName,List<Path>> reportTablesMissingRegions(final List<String> n
     return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA);
   }
 
+  public Map<TableName, List<byte[]>>
+  reportDirtyMetadata() throws IOException {
+    return HBCKMetaTableAccessor.getDirtyMetadata(this.conn);
+  }
+
+
+  public void deleteDirtyMetadata(Map<TableName, List<byte[]>> reportMap) throws IOException {
+    Table table = conn.getTable(TableName.META_TABLE_NAME);
+    for (Map.Entry<TableName, List<byte[]>> entry : reportMap.entrySet()) {
+      List<Delete> list=new ArrayList<>();
+      for (byte[] bytes : entry.getValue()) {
+        Delete delete=new Delete(bytes);
+        list.add(delete);
+      }
+      table.delete(list);
+      LOG.info("delete dirty table {} metadata",entry.getKey().getNameAsString());
+    }
+    table.close();
+  }
+
   List<Path> findMissingRegionsInMETA(String table) throws IOException {
     InternalMetaChecker<Path> missingChecker = new InternalMetaChecker<>();
     return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> {

hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java

@@ -23,6 +23,7 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.lang.reflect.Method;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -62,6 +63,7 @@
 import org.apache.hadoop.hbase.filter.SubstringComparator;
 import org.apache.hadoop.hbase.master.RegionState;
 
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.slf4j.Logger;
@@ -100,6 +102,7 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool {
   private static final String SET_REGION_STATE = "setRegionState";
   private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries";
   private static final String GENERATE_TABLE_INFO = "generateMissingTableDescriptorFile";
+  private static final String REPORT_DIRTY_METADATA = "reportDirtyMetadata";
   private static final String FIX_META = "fixMeta";
   // TODO update this map in case of the name of a method changes in Hbck interface
   //  in org.apache.hadoop.hbase.client package. Or a new command is added and the hbck command
@@ -270,6 +273,33 @@ Map<TableName, List<String>> extraRegionsInMeta(String[] args)
     return result;
   }
 
+  Map<TableName, List<String>> reportDirtyRegionsInMeta(String[] args)
+      throws Exception {
+    Options options = new Options();
+    Option fixOption = Option.builder("f").longOpt("fix").build();
+    options.addOption(fixOption);
+    // Parse command-line.
+    CommandLineParser parser = new DefaultParser();
+    CommandLine commandLine;
+    commandLine = parser.parse(options, args, false);
+    boolean fix = commandLine.hasOption(fixOption.getOpt());
+    Map<TableName, List<String>> result = new HashMap<>();
+    try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer =
+             new FsRegionsMetaRecoverer(this.conf)) {
+      Map<TableName, List<byte[]>> reportMap = fsRegionsMetaRecoverer.reportDirtyMetadata();
+      reportMap.forEach((key, value) -> result.put(key, value.stream().map(Bytes::toString)
+          .collect(Collectors.toList())));
+      if(fix) {
+        fsRegionsMetaRecoverer.deleteDirtyMetadata(reportMap);
+      }
+    } catch (IOException e) {
+      LOG.error("Error on checking extra regions: ", e);
+      throw e;
+    }
+
+    return result;
+  }
+
   private List<String> formatNameSpaceTableParam(String... nameSpaceOrTable) {
     return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null;
   }
@@ -412,6 +442,8 @@ private static String getCommandUsage() {
     writer.println();
     usageBypass(writer);
     writer.println();
+    usageReportDirtyRegionsInMeta(writer);
+    writer.println();
     usageExtraRegionsInMeta(writer);
     writer.println();
     usageFilesystem(writer);
@@ -558,6 +590,14 @@ private static void usageReplication(PrintWriter writer) {
     writer.println("   purge if '--fix'.");
   }
 
+  private static void usageReportDirtyRegionsInMeta(PrintWriter writer){
+    writer.println(" " + REPORT_DIRTY_METADATA + " [OPTIONS]");
+    writer.println("   Options:");
+    writer.println("    -f, --fix    fix meta by delete all dirty metadata found.");
+    writer.println("   Looks for undeleted metadata in meta table. ");
+    writer.println("   Undeleted metadata means a table has been deleted, but not delete all metadata in meta.");
+  }
+
   private static void usageExtraRegionsInMeta(PrintWriter writer) {
     writer.println(" " + EXTRA_REGIONS_IN_META + " <NAMESPACE|"
       + "NAMESPACE:TABLENAME>...");
@@ -945,6 +985,14 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc
           return EXIT_FAILURE;
         }
         break;
+      case REPORT_DIRTY_METADATA:
+        try {
+          Map<TableName, List<String>> report = reportDirtyRegionsInMeta(purgeFirst(commands));
+          System.out.println(formatDirtyMetadataReport(report));
+        }catch (Exception e) {
+          return EXIT_FAILURE;
+        }
+        break;
 
       case GENERATE_TABLE_INFO:
         if(commands.length != 2) {
@@ -978,6 +1026,11 @@ private String formatExtraRegionsReport(Map<TableName,List<String>> report) {
     return formatReportMessage(message, (HashMap)report, s -> s);
   }
 
+  private String formatDirtyMetadataReport(Map<TableName,List<String>> report) {
+    String message = "Dirty metadata in Meta, for each table:\n\t";
+    return formatReportMessage(message, (HashMap)report, s -> s);
+  }
+
   private String formatReportMessage(String reportMessage, Map<TableName, List<?>> report,
       Function resolver){
     final StringBuilder builder = new StringBuilder();

hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java

@@ -23,13 +23,7 @@
 import static org.apache.hadoop.hbase.HConstants.TABLE_STATE_QUALIFIER;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.SortedMap;
+import java.util.*;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.regex.Matcher;
@@ -60,6 +54,8 @@
 import org.apache.hadoop.hbase.client.TableState;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Pair;
@@ -263,6 +259,39 @@ public static List<RegionInfo> getAllRegions(Connection conn) throws IOException
       });
   }
 
+
+  /**
+   * List all dirty metadata currently in META.
+   * @param conn a valid, open connection.
+   * @return a Map of all dirty metadata in META.
+   * @throws IOException on any issues related with scanning meta table
+   */
+  public static Map<TableName, List<byte[]>> getDirtyMetadata(Connection conn) throws IOException {
+    Map<TableName, List<byte[]>> dirtyTableRegions = new HashMap<>();
+    Map<String, TableName> tableNameMap = new HashMap<>();
+    getTables(conn).forEach(tableName -> tableNameMap.put(tableName.getNameAsString(), tableName));
+    Table metaTable = conn.getTable(TableName.META_TABLE_NAME);
+    Scan scan = new Scan();
+    ResultScanner resultScanner = metaTable.getScanner(scan);
+    for (Result result : resultScanner) {
+      result.listCells().forEach(cell -> {
+        byte[] rowBytes = CellUtil.cloneRow(cell);
+        String row = Bytes.toString(rowBytes);
+        String tableName = row.split(",")[0];
+        if (!tableNameMap.containsKey(tableName)) {
+          if (dirtyTableRegions.containsKey(tableNameMap.get(tableName))) {
+            dirtyTableRegions.get(tableNameMap.get(tableName)).add(rowBytes);
+          } else {
+            List<byte[]> list = new ArrayList<>();
+            list.add(rowBytes);
+            dirtyTableRegions.put(tableNameMap.get(tableName), list);
+          }
+        }
+      });
+    }
+    return dirtyTableRegions;
+  }
+
   /**
    * Scans all "table:state" cell values existing in meta and returns as a map of
    * <code>TableName</code> as key and <code>TableState</code> as the value.