Adding ability to repair PageDB as a separate utility that can

be run outside the context of a hydra job. Moving class from
/test to /main in order to make it possible to deploy in executable
jar.

hydra-data/src/main/java/com/addthis/hydra/data/tree/ConcurrentTree.java

@@ -923,14 +923,10 @@ ConcurrentTreeNode getTreeTrashNode() {
         return treeTrashNode;
     }
 
-    /**
-     * For testing purposes only.
-     */
-    @SuppressWarnings("unused")
-    void testIntegrity() {
+    void repairIntegrity() {
         PagedKeyValueStore store = source.getEps();
         if (store instanceof SkipListCache) {
-            ((SkipListCache) store).testIntegrity(false);
+            ((SkipListCache) store).testIntegrity(true);
         }
     }
 }

hydra-data/src/main/java/com/addthis/hydra/data/tree/TreeIntegrity.java

@@ -0,0 +1,38 @@
+package com.addthis.hydra.data.tree;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TreeIntegrity {
+
+    private static final Logger log = LoggerFactory.getLogger(TreeIntegrity.class);
+
+    public static void main(String[] args) {
+        File root = new File(args[0]);
+        if (args.length > 1 && args[1].equals("repair")) {
+            ConcurrentTree tree = null;
+            try {
+                tree = new ConcurrentTree(root, false);
+                tree.repairIntegrity();
+            } catch (Exception ex) {
+                log.error(ex.toString());
+            } finally {
+                if (tree != null) tree.close();
+            }
+        } else {
+            ReadTree tree = null;
+            try {
+                tree = new ReadTree(root);
+                tree.testIntegrity();
+            } catch(Exception ex) {
+                log.error(ex.toString());
+            } finally {
+                if (tree != null) tree.close();
+            }
+        }
+    }
+
+
+}

hydra-store/src/main/java/com/addthis/hydra/store/kv/ReadExternalPagedStore.java

@@ -715,6 +715,7 @@ public ExternalPagedStoreMetrics getMetrics() {
 
     public void testIntegrity() {
         int counter = 0;
+        int failedPages = 0;
         try {
             byte[] encodedKey = pages.firstKey();
             K key = keyCoder.keyDecode(encodedKey);
@@ -725,12 +726,14 @@ public void testIntegrity() {
                     K nextKey = keyCoder.keyDecode(encodedNextKey);
                     K nextFirstKey = newPage.getNextFirstKey();
                     if (nextFirstKey == null) {
+                        failedPages++;
                         log.warn("On page " + counter + " the firstKey is " +
                              newPage.getFirstKey() +
                              " the nextFirstKey is null" +
                              " and the next page is associated with key " + nextKey);
                         assert(false);
                     } else if (!nextFirstKey.equals(nextKey)) {
+                        failedPages++;
                         int compareTo = compareKeys(nextFirstKey, nextKey);
                         char direction = compareTo > 0 ? '>' : '<';
                         log.warn("On page " + counter + " the firstKey is " +
@@ -744,13 +747,13 @@ public void testIntegrity() {
                 encodedKey = encodedNextKey;
                 counter++;
                 if (counter % 10000 == 0) {
-                    log.info("Scanned " + counter + " pages.");
+                    log.info("Scanned " + counter + " pages. Detected " + failedPages + " failed pages.");
                 }
             } while (encodedKey != null);
         } catch (ExecutionException ex) {
             log.error(ex.toString());
         }
-        log.info("Scanned " + counter + " pages.");
+        log.info("Scan complete. Scanned " + counter + " pages. Detected " + failedPages + " failed pages.");
     }
 
 }

hydra-store/src/main/java/com/addthis/hydra/store/skiplist/SkipListCache.java

@@ -2259,7 +2259,7 @@ public int testIntegrity(boolean repair) {
             }
         }
         log.info("Scan complete. Scanned " + counter + " pages. Detected " + failedPages + " failed pages.");
-        return failedPages;
+        return repair ? 0 : failedPages;
     }
 
 }