Split export categories from download categories.

java/db4e/controller/Controller.java

@@ -73,6 +73,7 @@ public class Controller {
    private boolean hasReset = false;
 
    public final ObservableList<Category> categories = new ObservableArrayList<>();
+   public final ObservableList<Category> exportCategories = new ObservableArrayList<>();
 
    private final SceneMain gui;
    private final ConsoleWebView browser;
@@ -427,14 +428,14 @@ public CompletableFuture<Void> startExport ( File target ) {
          log.log( Level.CONFIG, "Export root: {0}", target );
          new File( root ).mkdirs();
 
-         Convertor.beforeConvert( categories );
-
-         checkStop( "Writing main catlog" );
-         exporter.writeCatalog( root, categories );
-
          checkStop( "Loading content" );
          dal.loadEntityContent( categories, state );
 
+         checkStop( "Writing main catlog" );
+         Convertor.beforeConvert( categories, exportCategories );
+         exporter.writeCatalog( root, exportCategories );
+         state.total = exportCategories.stream().mapToInt( e -> e.getExportCount() ).sum();
+
          checkStop( "Converting" );
          convertDataForExport();
          Convertor.afterConvert();
@@ -450,7 +451,7 @@ public CompletableFuture<Void> startExport ( File target ) {
    }
 
    private void convertDataForExport () throws Exception {
-      forEachCategory( ( category ) -> {
+      exportEachCategory( ( category ) -> {
             Convertor convertor = Convertor.getConvertor( category, gui.isDebugging() );
             if ( convertor != null )
                return convertor.convert( state, threadPool );
@@ -459,7 +460,7 @@ private void convertDataForExport () throws Exception {
    }
 
    private void exportData ( String root ) throws Exception {
-      forEachCategory( ( category ) -> {
+      exportEachCategory( ( category ) -> {
          log.log( Level.FINE, "Writing {0}", category.id );
          return exporter.writeCategory( root, category, state, threadPool );
       }, Exporter.stop );
@@ -562,14 +563,14 @@ private void runAndGet ( String taskName, RunExcept task ) {
       } while ( true );
    }
 
-   private void forEachCategory ( FunctionExcept<Category, CompletableFuture<Void>> task, AtomicBoolean stop ) throws Exception {
+   private void exportEachCategory ( FunctionExcept<Category, CompletableFuture<Void>> task, AtomicBoolean stop ) throws Exception {
       state.reset();
       state.update();
       log.log( Level.CONFIG, "Running category task in {0} threads.", threadPool.getParallelism() );
       try {
          stop.set( false );
          List<CompletableFuture<Void>> tasks = new ArrayList<>( categories.size() );
-         categories.stream().sorted( (a,b) -> b.getExportCount() - a.getExportCount() ).forEachOrdered( ( category ) -> { try {
+         exportCategories.stream().sorted( (a,b) -> b.getExportCount() - a.getExportCount() ).forEachOrdered( ( category ) -> { try {
             CompletableFuture<Void> future = task.apply( category );
             if ( future != null ) tasks.add( future );
          } catch ( Exception e ) {

java/db4e/controller/Exporter.java

@@ -30,25 +30,20 @@ void writeCatalog ( String target, List<Category> categories ) throws IOExceptio
       try ( OutputStreamWriter writer = openStream( target + "/catalog.js" ) ) {
          buffer.append( "od.reader.jsonp_catalog(20130616,{" );
          for ( Category category : categories ) {
-            if ( category.getExportCount() > 0 )
-               str( buffer, category.id.toLowerCase() ).append( ':' ).append( category.getExportCount() ).append( ',' );
+            str( buffer, category.id.toLowerCase() ).append( ':' ).append( category.getExportCount() ).append( ',' );
          }
          write( "})", writer, buffer );
       }
    }
 
 
    CompletableFuture<Void> writeCategory ( String target, Category category, ProgressState state, Executor pool ) throws IOException {
-      synchronized ( category ) {
-         if ( category.meta == null || category.total_entry.get() + category.exported_entry_deviation.get() <= 0 )
-            return CompletableFuture.completedFuture( null );
-      }
       final CompletableFuture<Void> result = new CompletableFuture();
 
       pool.execute( () -> { try { synchronized( category ) {
          if ( stop.get() ) throw new InterruptedException();
          String cat_id = category.id.toLowerCase();
-         
+
          StringBuilder buffer = new StringBuilder( 1024 );
          File catPath = new File( target + "/" + cat_id + "/" );
          catPath.mkdir();
@@ -73,36 +68,35 @@ CompletableFuture<Void> writeCategory ( String target, Category category, Progre
             write( ",{", index, buffer );
 
             for ( Entry entry : category.sorted ) {
-               if ( ! "null".equals( entry.shortid ) ) {
-                  // Add to listing
-                  str( buffer.append( '[' ), entry.shortid ).append( ',' );
-                  str( buffer, entry.display_name ).append( ',' );
-                  for ( Object field : entry.meta )
-                     str( buffer, field.toString() ).append( ',' );
-                  write( "],", listing, buffer );
-
-                  // Add to full text
-                  str( buffer, entry.shortid ).append( ':' );
-                  str( buffer, entry.fulltext );
-                  write( ",", index, buffer );
-
-                  // Group content
-                  if ( ! regxIdGroup.reset( entry.shortid ).find() )
-                     throw new IllegalStateException( "Invalid id " + entry.shortid );
-                  int grp = Integer.parseUnsignedInt( regxIdGroup.group( 2 ) );
-
-                  // Write content
-                  if ( writers[ grp ] == null ) {
-                     writers[ grp ] = openStream( catPath + "/data" + grp + ".js" );
-                     buffer.append( "od.reader.jsonp_batch_data(20160803," );
-                     str( buffer, cat_id );
-                     write( ",{", writers[grp], buffer );
-                  }
-                  str( buffer, entry.shortid ).append( ':' );
-                  str( buffer, entry.data );
-                  write( ",", writers[grp], buffer );
-                  ++exported;
+               // Add to listing
+               str( buffer.append( '[' ), entry.shortid ).append( ',' );
+               str( buffer, entry.display_name ).append( ',' );
+               for ( Object field : entry.meta )
+                  str( buffer, field.toString() ).append( ',' );
+               write( "],", listing, buffer );
+
+               // Add to full text
+               str( buffer, entry.shortid ).append( ':' );
+               str( buffer, entry.fulltext );
+               write( ",", index, buffer );
+
+               // Group content
+               if ( ! regxIdGroup.reset( entry.shortid ).find() )
+                  throw new IllegalStateException( "Invalid id " + entry.shortid );
+               int grp = Integer.parseUnsignedInt( regxIdGroup.group( 2 ) );
+
+               // Write content
+               if ( writers[ grp ] == null ) {
+                  writers[ grp ] = openStream( catPath + "/data" + grp + ".js" );
+                  buffer.append( "od.reader.jsonp_batch_data(20160803," );
+                  str( buffer, cat_id );
+                  write( ",{", writers[grp], buffer );
                }
+               str( buffer, entry.shortid ).append( ':' );
+               str( buffer, entry.data );
+               write( ",", writers[grp], buffer );
+               ++exported;
+
                if ( stop.get() ) throw new InterruptedException();
                state.addOne();
             }

java/db4e/convertor/Convertor.java

@@ -6,6 +6,7 @@
 import db4e.data.Entry;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -27,28 +28,48 @@ public abstract class Convertor {
    public static AtomicBoolean stop = new AtomicBoolean();
 
    protected final Category category;
-   protected static final Map<String,Category> categories = new HashMap<>( 18, 1.0f );
 
    /**
     * Called before doing any export.
     * Can be used to fix entry count before catalog is saved.
     *
     * @param categories
     */
-   public static void beforeConvert ( List<Category> categories ) {
-      synchronized ( Convertor.categories ) {
-         if ( Convertor.categories.isEmpty() ) {
-            for ( Category c : categories )
-               Convertor.categories.put( c.id, c );
-            int terrainCount = Convertor.categories.get( "Terrain" ).total_entry.get();
-            Convertor.categories.get( "Terrain" ).exported_entry_deviation.set( -terrainCount );
-            Convertor.categories.get( "Trap" ).exported_entry_deviation.set( terrainCount );
-            Convertor.categories.get( "Glossary" ).exported_entry_deviation.set( -79 );
-         }
-      }
+   public static void beforeConvert ( List<Category> categories, List<Category> exportCategories ) {
+      if ( exportCategories.size() > 0 ) return;
       synchronized( corrected ) {
          corrected.clear();
       }
+      synchronized ( categories ) {
+         Map<String,Category> map = new HashMap<>( 18, 1.0f );
+         exportCategories.clear();
+         if ( map.isEmpty() ) {
+            for ( Category c : categories )
+               map.put( c.id, c );
+
+            for ( Category c : categories ) {
+               if ( c.id.equals( "Terrain" ) ) continue;
+               Category exported = new Category( c.id, c.name, c.type, c.fields );
+               exportCategories.add( exported );
+               exported.entries.addAll( c.entries );
+               switch ( c.id ) {
+                  case "Glossary" :
+                     for ( Iterator<Entry> i = exported.entries.iterator() ; i.hasNext() ; ) {
+                        Entry entry = i.next();
+                        // Various empty glossaries. Such as "male" or "female".  glossary679 "familiar" does not even have published.
+                        if ( entry.id.equals( "glossary.aspx?id=679" ) || entry.content.contains( "</h1><p class=\"flavor\"></p><p class=\"publishedIn\">" ) ) {
+                           i.remove();
+                           corrected( entry, "blacklist" );
+                        }
+                     }
+                     break;
+
+                  case "Trap" :
+                     exported.entries.addAll( map.get( "Terrain" ).entries );
+               }
+            }
+         }
+      }
    }
 
    public static void afterConvert () {
@@ -96,10 +117,13 @@ public CompletableFuture<Void> convert ( ProgressState state, Executor pool ) {
          if ( category.meta == null )
             category.meta = category.fields;
          initialise();
-         final List<Entry> entries = getExportEntries();
+         final List<Entry> entries = category.entries;
          for ( Entry entry : entries ) {
-            if ( entry.content == null ) throw new IllegalStateException( entry.name + " (" + category.name + ") has no content" );
-            convertEntry( entry );
+            if ( entry.fulltext == null ) {
+               if ( entry.content == null ) throw new IllegalStateException( "No content: " + entry.id + " " + entry.name );
+               convertEntry( entry );
+               if ( entry.fulltext == null ) throw new IllegalStateException( "No converted data: " + entry.name + " " + category.name );
+            }
             if ( stop.get() ) throw new InterruptedException();
             state.addOne();
          }
@@ -116,10 +140,6 @@ public CompletableFuture<Void> convert ( ProgressState state, Executor pool ) {
 
    protected void initialise()  { }
 
-   protected List<Entry> getExportEntries() {
-      return category.entries;
-   }
-
    protected int sortEntity ( Entry a, Entry b ) {
       return a.name.compareTo( b.name );
    }
@@ -136,15 +156,18 @@ protected void convertEntry ( Entry entry ) {
       copyMeta( entry );
       entry.data = normaliseData( entry.content );
       String fixApplied = correctEntry( entry );
-      if ( fixApplied != null ) synchronized ( corrected ) {
+      if ( fixApplied != null ) corrected(entry, fixApplied);
+      parseSourceBook( entry );
+      entry.fulltext = textData( entry.data );
+      // DefaultConvertor will do some checking if debug is on.
+   }
+
+   private static void corrected ( Entry entry, String fixApplied ) {
+      synchronized ( corrected ) {
          log.log( Level.FINE, "Corrected {0} {1} ({2})", new Object[]{ entry.shortid, entry.name, fixApplied });
          if ( corrected.containsKey( fixApplied ) ) corrected.get( fixApplied ).incrementAndGet();
          else corrected.put( fixApplied, new AtomicInteger( 1 ) );
       }
-      if ( "null".equals( entry.shortid ) ) return;
-      parseSourceBook( entry );
-      entry.fulltext = textData( entry.data );
-      // DefaultConvertor will do some checking if debug is on.
    }
 
    protected void copyMeta ( Entry entry ) {

java/db4e/convertor/DefaultConvertor.java

@@ -38,7 +38,6 @@ public DefaultConvertor ( Category category, boolean debug ) {
    @Override protected void convertEntry ( Entry entry ) {
       super.convertEntry( entry );
       if ( debug ) {
-         if ( "null".equals( entry.shortid ) ) return;
          // These checks are enabled only when debug log is showing, mainly for development and debug purpose.
          if ( shortId.containsKey( entry.shortid ) )
             log.log( Level.WARNING, "{1} duplicate shortid '{2}': {3} & {0}", new Object[]{ entry.id, entry.name, entry.shortid, shortId.get( entry.shortid ).name } );

java/db4e/convertor/FieldSortConvertor.java

@@ -24,12 +24,7 @@ public FieldSortConvertor(Category category, int sort_field, boolean debug) {
    @Override protected String correctEntry(Entry entry) {
       switch ( category.id ) {
       case "Glossary":
-         // Various empty glossaries. Such as "male" or "female".  "familiar" does not even have text in publish block.
-         if ( entry.data.contains( "</h1><p class=flavor></p><p class=publishedIn>" ) ) {
-            entry.shortid = "null"; // Just blacklist them and forget they ever existed.
-            return "blacklist";
-
-         } else if ( entry.shortid.startsWith( "skill" ) ) { // Fix skills missing "improvising with" title
+         if ( entry.shortid.startsWith( "skill" ) ) { // Fix skills missing "improvising with" title
             if ( entry.data.contains( "<p class=flavor><b></b></p><p class=flavor>" ) ) {
                entry.data = entry.data.replace( "<p class=flavor><b></b></p><p class=flavor>", "<h3>IMPROVISING WITH "+entry.name.toUpperCase()+"</h3><p class=flavor>" );
                return "missing subtitle";

java/db4e/convertor/TrapConvertor.java

@@ -2,24 +2,14 @@
 
 import db4e.data.Category;
 import db4e.data.Entry;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
 public class TrapConvertor extends LeveledConvertor {
 
    public TrapConvertor ( Category category, boolean debug ) {
       super( category, debug );
    }
 
-   @Override protected List<Entry> getExportEntries() {
-      Category terrain = categories.get( "Terrain" );
-      List<Entry> result = new ArrayList<>( category.entries.size() + terrain.total_entry.get() );
-      result.addAll( category.entries );
-      result.addAll( terrain.entries );
-      return result;
-   }
-
    @Override protected String correctEntry (Entry entry) {
       if ( entry.meta.length == 4 ) { // Trap
          // 7 traps in Dungeon 214-215 has level like "8 Minion" and no group role.

java/db4e/data/Category.java

@@ -16,10 +16,6 @@ public class Category {
    public final IntegerProperty total_entry = new SimpleIntegerProperty();
    // Number of entry with downloaded content. Will increase during the download process.
    public final IntegerProperty downloaded_entry = new SimpleIntegerProperty();
-   // Negative if some in-database entries are blacklisted and won't be exported.
-   // Positive if some non-database entries are added into this category.
-   // Must be set before exporting.
-   public final IntegerProperty exported_entry_deviation = new SimpleIntegerProperty();
 
    public final String[] fields; // Name (id) of compendium fields
    public final List<Entry> entries = new ArrayList<>(); // Entry list
@@ -34,17 +30,11 @@ public Category( String id, String name, String type, String[] fields ) {
       this.fields = fields;
    }
 
-   @Override public String toString () {
-      StringBuilder str = new StringBuilder().append( id ).append( ' ' );
-      str.append( entries.size() ).append( '/' ).append( total_entry );
-      return str.toString();
-   }
-
    public String getName() { return name; }
    public IntegerProperty totalEntryProperty() { return total_entry; }
    public IntegerProperty downloadedEntryProperty() { return downloaded_entry; }
 
    public int getExportCount () {
-      return total_entry.get() + exported_entry_deviation.get();
+      return entries.size();
    }
 }

java/db4e/data/Entry.java

@@ -12,7 +12,7 @@ public class Entry {
 
    // Transformed data for export
    public String display_name; // Converted name for export
-   public String shortid;     // Simplified id for export. Set to "null" (String) to skip export.
+   public String shortid;     // Simplified id for export
    public String fulltext;   // Full text index text - without name and flavour
    public Object[] meta;    // Transform field data
    public String data;     // Processed data text