Slightly more robust page linking, still plenty to do however

.gitignore

@@ -6,4 +6,6 @@
target

graph-db/*
resources/tbontb-regular.pdf
resources/*

itext-rups/*

graphstyle.grass

@@ -0,0 +1,84 @@
node {
  diameter: 50px;
  color: #A5ABB6;
  border-color: #9AA1AC;
  border-width: 2px;
  text-color-internal: #FFFFFF;
  font-size: 10px;
  caption: 'Page {page_number}';
}

relationship {
  color: #A5ABB6;
  shaft-width: 1px;
  font-size: 8px;
  padding: 3px;
  text-color-external: #000000;
  text-color-internal: #FFFFFF;
  caption: '<type>';
}

node.Page {
  color: #68BDF6;
  border-color: #5CA8DB;
  text-color-internal: #FFFFFF;
  caption: '{book_page_label}';
}

node.Ending {
  color: #6DCE9E;
  border-color: #60B58B;
  text-color-internal: #FFFFFF;
  caption: '{page_number}';
}

node.EndPage {
  color: #FF756E;
  border-color: #E06760;
  text-color-internal: #FFFFFF;
}

node.Ignore {
  color: #A5ABB6;
  border-color: #9AA1AC;
  text-color-internal: #FFFFFF;
}

node.ImagePage {
  color: #6DCE9E;
  border-color: #60B58B;
  text-color-internal: #FFFFFF;
}

node.SubBook {
  color: #FFD86E;
  border-color: #EDBA39;
  text-color-internal: #604A0E;
  diameter: 50px;
}

node.SubBook.Ending {
  color: #6DCE9E;
  border-color: #60B58B;
  text-color-internal: #FFFFFF;
  caption: '{page_number}';
}

node.SubBook.EndPage {
  color: #FF756E;
  border-color: #E06760;
  text-color-internal: #FFFFFF;
}

node.SubBook.Ignore {
  color: #A5ABB6;
  border-color: #9AA1AC;
  text-color-internal: #FFFFFF;
}

node.SubBook.ImagePage {
  color: #6DCE9E;
  border-color: #60B58B;
  text-color-internal: #FFFFFF;
}

src/main/java/uk/co/palmr/App.java

@@ -1,113 +1,229 @@
package uk.co.palmr;

import com.itextpdf.text.pdf.PdfReader;
import com.itextpdf.text.pdf.parser.PdfReaderContentParser;
import com.itextpdf.text.pdf.parser.SimpleTextExtractionStrategy;
import com.itextpdf.text.pdf.parser.TextExtractionStrategy;
import com.itextpdf.text.factories.RomanAlphabetFactory;
import com.itextpdf.text.factories.RomanNumberFactory;
import com.itextpdf.text.pdf.*;
import com.itextpdf.text.pdf.parser.*;
import org.neo4j.graphdb.*;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.register.Register;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class App {
  private enum RelationshipTypes implements RelationshipType {
    ContinuesTo,
    RedirectsTo
    Continues,
    Choice
  }

  private enum NodeTypes implements Label {
  private enum PageTypes implements Label {
    Page,
    Ending
    ImagePage,
    EndPage,
    SubBook,
    Ignore
  }

  private static final int PAGE_OFFSET = 4;
  private static final int SUB_BOOK_OFFSET = 24;
  private static final String PDF_PAGE_NUMBER = "pdf_page_number";
  private static final String BOOK_PAGE_LABEL = "book_page_label";

  public static void main(String[] args) throws IOException {
    PdfReader reader = new PdfReader("C:\\Users\\npalmer\\git-projects\\Adventurer\\resources\\tbontb-regular.pdf");
    PdfReader reader = new PdfReader(App.class.getClassLoader().getResource("").getPath() + "\\..\\..\\resources\\tbontb-regular.pdf");

    GraphDatabaseService graphDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder("C:\\Users\\npalmer\\git-projects\\Adventurer\\graph-db").newGraphDatabase();
    GraphDatabaseService graphDb = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(App.class.getClassLoader().getResource("").getPath() + "\\..\\..\\graph-db").newGraphDatabase();
    registerShutdownHook(graphDb);

    System.out.println("Attempting to clear database from last run");
    try (Transaction ignored = graphDb.beginTx();
         Result result = graphDb.execute("MATCH (n)\n" +
           "OPTIONAL MATCH (n)-[r]-()\n" +
           "DELETE n,r")) {
      System.out.println(result.resultAsString());
      ignored.success();
    }


    System.out.println("Attempting to populate database");
    try (Transaction tx = graphDb.beginTx()) {
      long[] pageNodeIdArray = new long[reader.getNumberOfPages() + 1];
      for (int page = 1 + PAGE_OFFSET; page <= reader.getNumberOfPages(); page++) {
        Node pageNode = graphDb.createNode(NodeTypes.Page);
        pageNode.setProperty("page_number", resolveBookPageNumber(page));
        pageNode.setProperty("pdf_page_number", page);
        pageNodeIdArray[page] = pageNode.getId();
      System.out.println("Creating page nodes");
      String[] bookPageLabels = getPageLabels(reader);
      Map<PdfObject, String> pdfPageToBookPageLabel = new HashMap<>(reader.getNumberOfPages());
      for (int pdfPageNumber = 1; pdfPageNumber <= reader.getNumberOfPages(); pdfPageNumber++) {
        Node pageNode = graphDb.createNode(PageTypes.Page);
        pageNode.setProperty(PDF_PAGE_NUMBER, pdfPageNumber);
        pageNode.setProperty(BOOK_PAGE_LABEL, bookPageLabels[pdfPageNumber - 1]);
        pdfPageToBookPageLabel.put(reader.getPageN(pdfPageNumber), bookPageLabels[pdfPageNumber - 1]);
      }

      PdfReaderContentParser parser = new PdfReaderContentParser(reader);
      TextExtractionStrategy strategy;
      for (int page = 1 + PAGE_OFFSET; page <= reader.getNumberOfPages(); page++) {
        strategy = parser.processContent(page, new SimpleTextExtractionStrategy());
        System.out.println(strategy.getResultantText());
        System.out.println("");
      System.out.println("Looping over pages");
      PdfReaderContentParser contentParser = new PdfReaderContentParser(reader);
      Map<String, PdfObject> linkDestinations = reader.getNamedDestinationFromStrings();

        Node thisPage = graphDb.getNodeById(pageNodeIdArray[page]);
      for (int pdfPageNumber = 1; pdfPageNumber <= reader.getNumberOfPages(); pdfPageNumber++) {
//      int pdfPageNumber = 433;
//      {
        Node thisPage = graphDb.findNode(PageTypes.Page, PDF_PAGE_NUMBER, pdfPageNumber);
        if (thisPage == null) {
          throw new RuntimeException("Couldn't find a node for PDF page: " + pdfPageNumber);
        }

        if (strategy.getResultantText().contains("THE END")) {
          thisPage.removeLabel(NodeTypes.Page);
          thisPage.addLabel(NodeTypes.Ending);
          System.out.println("End Page");
        String thisBookPageLabel = (String)thisPage.getProperty(BOOK_PAGE_LABEL);
        boolean isSubBook = false;
        if (thisBookPageLabel.startsWith("G")) {
          // All of the sub book (The Murder of Gonzago) page labels start with G
          thisPage.addLabel(PageTypes.SubBook);
        }
        if (!thisBookPageLabel.matches(".*\\d+.*")) {
          // If there's no digits in the label, ignore the page
          thisPage.addLabel(PageTypes.Ignore);
        }
        else {
          // TODO the choice part of the regex doesn't work multi-line properly, ideally use annotations and check for preceeding text blocks?
          Pattern pageLinkPattern = Pattern.compile("(.*?)turn to page ([0-9]+)", Pattern.CASE_INSENSITIVE);
          Matcher pageLinkMatcher = pageLinkPattern.matcher(strategy.getResultantText());
          boolean linksfound = false;
          while (pageLinkMatcher.find()) {
            System.out.println("Choice: " + pageLinkMatcher.group(1));
            System.out.println("Links to: " + pageLinkMatcher.group(2));

            Node targetPage = graphDb.getNodeById(pageNodeIdArray[resolvePDFPageNumber(Integer.valueOf(pageLinkMatcher.group(2)))]);

            Relationship relationship = thisPage.createRelationshipTo(targetPage, RelationshipTypes.RedirectsTo);
            relationship.setProperty("choice", pageLinkMatcher.group(1));
        // Check for inter-page-links
        PdfDictionary pageDict = reader.getPageN(pdfPageNumber);
        PdfArray annotArray = pageDict.getAsArray(PdfName.ANNOTS);
        boolean linksOut = false;
        if (annotArray != null) {
          for (int i = 0; i < annotArray.size(); i++) {
            PdfDictionary annotDict = annotArray.getAsDict(i);
            if (PdfName.LINK == annotDict.get(PdfName.SUBTYPE)) {
              if (annotDict.contains(PdfName.A)) {
                String lDestination = ((PdfDictionary) reader.getPdfObject(annotDict.getAsIndirectObject(PdfName.A))).getAsString(PdfName.D).toString();
                if (linkDestinations.containsKey(lDestination)) {
                  PdfArray destinationInfoArray = (PdfArray) linkDestinations.get(lDestination);
                  PdfIndirectReference destinationReference = destinationInfoArray.getAsIndirectObject(0); // TODO, this could actually be an integer for the case of Remote Destinations
                  PdfObject targetPdfPage = PdfReader.getPdfObject(destinationReference);

                  // Create link if it hasn't already been made between these pages (Split-line links mean two annots with the same dest)
                  Node targetPage = graphDb.findNode(PageTypes.Page, BOOK_PAGE_LABEL, pdfPageToBookPageLabel.get(targetPdfPage));
                  boolean existingRelationship = false;
                  for (Relationship r : thisPage.getRelationships(Direction.BOTH, RelationshipTypes.Choice)) {
                    existingRelationship |= r.getEndNode().getId() == targetPage.getId();
                  }
                  if (!existingRelationship) {
                    thisPage.createRelationshipTo(targetPage, RelationshipTypes.Choice);
                    linksOut = true;
                  }
                }
                else {
                  System.out.println("Found link to unknown: " + lDestination);
                }
              }
              else {
                System.out.println("Unhandled");
              }
            }
          }
        }

            System.out.println("Rel created from " + relationship.getStartNode().getProperty("page_number") + "[" + relationship.getStartNode().getProperty("pdf_page_number") + "] to " + relationship.getEndNode().getProperty("page_number") + "[" + relationship.getEndNode().getProperty("pdf_page_number") + "]");

            System.out.println("");
            linksfound = true;
        // Do some context scanning to find page types
        FontGroupingTextExtractionStrategy strategy = contentParser.processContent(pdfPageNumber, new FontGroupingTextExtractionStrategy(false));
        if (strategy.getImageCount() > 0 && (strategy.getTextValues().size() == 0 || (strategy.getTextValues().size() == 1 && strategy.getTextValues().get(0).toString().equals(bookPageLabels[pdfPageNumber])))) {
          // If the page is nothing but an image, label it as an image page (Typically an ending comic)
          thisPage.addLabel(PageTypes.ImagePage);
        }
        else {
          // If not an image page perhaps there's text to parse?
          for (StringBuilder textBlock : strategy.getTextValues()) {
            if (textBlock.toString().matches("THE END(!!)?")) {
              thisPage.addLabel(PageTypes.EndPage);
            }
          }
          if (!linksfound && page != reader.getNumberOfPages()) {
            Node targetPage = graphDb.getNodeById(pageNodeIdArray[page + 1]);
            thisPage.createRelationshipTo(targetPage, RelationshipTypes.ContinuesTo);
        }

        // Link to next page if no other relationships from this node
        if (!linksOut && !thisPage.hasLabel(PageTypes.EndPage)) {
          Node nextPage = graphDb.findNode(PageTypes.Page, PDF_PAGE_NUMBER, pdfPageNumber+1);
          if (nextPage != null) {
            thisPage.createRelationshipTo(nextPage, RelationshipTypes.Continues);
          }
        }
        System.out.println("");
        System.out.println("------------");
        System.out.println("");
      }
      reader.close();

      tx.success();
    }

    graphDb.shutdown();

    // Query e.g. MATCH (p:Page {page_number: 22})-[*..10]-(p2) RETURN p, p2
  }
    reader.close();
    graphDb.shutdown();

  private static int resolveBookPageNumber(int pdfPageNumber) {
    if (pdfPageNumber <= 414) {
      return pdfPageNumber - PAGE_OFFSET;
    }
    else {
      return pdfPageNumber - PAGE_OFFSET - SUB_BOOK_OFFSET;
    }
    // Query e.g. MATCH r=(s:SubBook {book_page_label: "Gi"})-[*..10]->(e:SubBook :EndPage) RETURN r
  }

  private static int resolvePDFPageNumber(int bookPageNumber) {
    if (bookPageNumber <= 414) {
      return bookPageNumber + PAGE_OFFSET;
    }
    else {
      return bookPageNumber + PAGE_OFFSET + SUB_BOOK_OFFSET;
  /**
   * Retrieves the page labels from a PDF as an array of String objects.
   * @param reader a PdfReader object that has the page labels you want to retrieve
   * @return	a String array or <code>null</code> if no page labels are present
   */
  public static String[] getPageLabels(PdfReader reader) {

    int n = reader.getNumberOfPages();

    PdfDictionary dict = reader.getCatalog();
    PdfDictionary labels = (PdfDictionary)PdfReader.getPdfObjectRelease(dict.get(PdfName.PAGELABELS));
    if (labels == null)
      return null;

    String[] labelstrings = new String[n];

    HashMap<Integer, PdfObject> numberTree = PdfNumberTree.readTree(labels);

    int pagecount = 1;
    Integer current;
    String prefix = "";
    char type = 'D';
    for (int i = 0; i < n; i++) {
      current = Integer.valueOf(i);
      if (numberTree.containsKey(current)) {
        PdfDictionary d = (PdfDictionary)PdfReader.getPdfObjectRelease(numberTree.get(current));
        if (d.contains(PdfName.ST)) {
          pagecount = ((PdfNumber)d.get(PdfName.ST)).intValue();
        }
        else {
          pagecount = 1;
        }
        if (d.contains(PdfName.P)) {
          prefix = ((PdfString)d.get(PdfName.P)).toUnicodeString();
        }
        else {
          prefix = ""; // NP - See page 596 of pdf ref, prefix is for that range only
        }
        if (d.contains(PdfName.S)) {
          type = ((PdfName)d.get(PdfName.S)).toString().charAt(1);
        }
        else {
          type = 'e';
        }
      }
      switch(type) {
        default:
          labelstrings[i] = prefix + pagecount;
          break;
        case 'R':
          labelstrings[i] = prefix + RomanNumberFactory.getUpperCaseString(pagecount);
          break;
        case 'r':
          labelstrings[i] = prefix + RomanNumberFactory.getLowerCaseString(pagecount);
          break;
        case 'A':
          labelstrings[i] = prefix + RomanAlphabetFactory.getUpperCaseString(pagecount);
          break;
        case 'a':
          labelstrings[i] = prefix + RomanAlphabetFactory.getLowerCaseString(pagecount);
          break;
        case 'e':
          labelstrings[i] = prefix;
          break;
      }
      pagecount++;
    }
    return labelstrings;
  }

  private static void registerShutdownHook(final GraphDatabaseService graphDb) {