Permalink
Browse files

Work on #242. This closes #186, closes #175, closes #185, closes 161,

closes #166, closes #154, closes #135, closes #138 and closes #104.
  • Loading branch information...
jonbullock committed Nov 18, 2015
2 parents 84b8f3d + 3e9d8a3 commit 698cd7a9eb552b68432b1198f07e829369950580
Showing with 3,601 additions and 1,581 deletions.
  1. +391 −310 pom.xml
  2. +19 −0 src/main/assembly/assembly-example-project-jade.xml
  3. +19 −0 src/main/assembly/assembly-example-project-pebble.xml
  4. +16 −3 src/main/java/org/jbake/app/ConfigUtil.java
  5. +56 −2 src/main/java/org/jbake/app/ContentStore.java
  6. +69 −25 src/main/java/org/jbake/app/Crawler.java
  7. +11 −4 src/main/java/org/jbake/app/DBUtil.java
  8. +10 −55 src/main/java/org/jbake/app/Oven.java
  9. +14 −208 src/main/java/org/jbake/app/Parser.java
  10. +182 −97 src/main/java/org/jbake/app/Renderer.java
  11. +9 −0 src/main/java/org/jbake/model/DocumentTypes.java
  12. +10 −2 src/main/java/org/jbake/parser/AsciidoctorEngine.java
  13. +10 −10 src/main/java/org/jbake/parser/Engines.java
  14. +216 −1 src/main/java/org/jbake/parser/MarkupEngine.java
  15. +5 −5 src/main/java/org/jbake/parser/ParserContext.java
  16. +12 −0 src/main/java/org/jbake/parser/ParserEngine.java
  17. +29 −0 src/main/java/org/jbake/render/ArchiveRenderer.java
  18. +44 −0 src/main/java/org/jbake/render/DocumentsRenderer.java
  19. +29 −0 src/main/java/org/jbake/render/FeedRenderer.java
  20. +29 −0 src/main/java/org/jbake/render/IndexRenderer.java
  21. +14 −0 src/main/java/org/jbake/render/RenderingTool.java
  22. +29 −0 src/main/java/org/jbake/render/SitemapRenderer.java
  23. +30 −0 src/main/java/org/jbake/render/TagsRenderer.java
  24. +6 −5 src/main/java/org/jbake/template/AbstractTemplateEngine.java
  25. +24 −52 src/main/java/org/jbake/template/FreemarkerTemplateEngine.java
  26. +86 −0 src/main/java/org/jbake/template/GroovyMarkupTemplateEngine.java
  27. +7 −59 src/main/java/org/jbake/template/GroovyTemplateEngine.java
  28. +116 −0 src/main/java/org/jbake/template/JadeTemplateEngine.java
  29. +18 −0 src/main/java/org/jbake/template/ModelExtractor.java
  30. +154 −0 src/main/java/org/jbake/template/ModelExtractors.java
  31. +17 −0 src/main/java/org/jbake/template/NoModelExtractorException.java
  32. +127 −0 src/main/java/org/jbake/template/PebbleTemplateEngine.java
  33. +27 −0 src/main/java/org/jbake/template/TemplateEngineAdapter.java
  34. +11 −9 src/main/java/org/jbake/template/TemplateEngines.java
  35. +9 −60 src/main/java/org/jbake/template/ThymeleafTemplateEngine.java
  36. +27 −0 src/main/java/org/jbake/template/model/AllContentExtractor.java
  37. +29 −0 src/main/java/org/jbake/template/model/AllTagsExtractor.java
  38. +15 −0 src/main/java/org/jbake/template/model/DBExtractor.java
  39. +16 −0 src/main/java/org/jbake/template/model/ModelExtractionUtils.java
  40. +27 −0 src/main/java/org/jbake/template/model/PublishedContentExtractor.java
  41. +16 −0 src/main/java/org/jbake/template/model/PublishedDateExtractor.java
  42. +20 −0 src/main/java/org/jbake/template/model/PublishedPagesExtractor.java
  43. +20 −0 src/main/java/org/jbake/template/model/PublishedPostsExtractor.java
  44. +26 −0 src/main/java/org/jbake/template/model/TagPostsExtractor.java
  45. +27 −0 src/main/java/org/jbake/template/model/TypedDocumentsExtractor.java
  46. +5 −1 src/main/resources/META-INF/org.jbake.parser.TemplateEngines.properties
  47. +9 −0 src/main/resources/META-INF/org.jbake.template.ModelExtractors.properties
  48. +7 −0 src/main/resources/META-INF/services/org.jbake.render.RenderingTool.properties
  49. +5 −1 src/main/resources/default.properties
  50. +60 −3 src/test/java/org/jbake/app/CrawlerTest.java
  51. +101 −0 src/test/java/org/jbake/app/PaginationTest.java
  52. +0 −228 src/test/java/org/jbake/app/RendererTest.java
  53. +0 −215 src/test/java/org/jbake/app/ThymeleafRendererTest.java
  54. +108 −72 .../java/org/jbake/app/{GroovyRendererTest.java → template/AbstractTemplateEngineRenderingTest.java}
  55. +81 −0 src/test/java/org/jbake/app/template/FreemarkerTemplateEngineRenderingTest.java
  56. +33 −0 src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java
  57. +61 −0 src/test/java/org/jbake/app/template/GroovyTemplateEngineRenderingTest.java
  58. +33 −0 src/test/java/org/jbake/app/template/JadeTemplateEngineRenderingTest.java
  59. +33 −0 src/test/java/org/jbake/app/template/PebbleTemplateEngineRenderingTest.java
  60. +60 −0 src/test/java/org/jbake/app/template/ThymeleafTemplateEngineRenderingTest.java
  61. +27 −0 src/test/resources/freemarkerTemplates/archive.ftl
  62. +6 −6 src/test/resources/{templates → freemarkerTemplates}/feed.ftl
  63. +19 −0 src/test/resources/freemarkerTemplates/footer.ftl
  64. +35 −0 src/test/resources/freemarkerTemplates/header.ftl
  65. +20 −0 src/test/resources/freemarkerTemplates/index.ftl
  66. +34 −0 src/test/resources/freemarkerTemplates/menu.ftl
  67. +15 −0 src/test/resources/freemarkerTemplates/page.ftl
  68. +15 −0 src/test/resources/freemarkerTemplates/post.ftl
  69. 0 src/test/resources/{templates → freemarkerTemplates}/sitemap.ftl
  70. +27 −0 src/test/resources/freemarkerTemplates/tags.ftl
  71. +9 −0 src/test/resources/groovyMarkupTemplates/allcontent.tpl
  72. +26 −0 src/test/resources/groovyMarkupTemplates/archive.tpl
  73. +22 −0 src/test/resources/groovyMarkupTemplates/feed.tpl
  74. +8 −0 src/test/resources/groovyMarkupTemplates/footer.tpl
  75. +59 −0 src/test/resources/groovyMarkupTemplates/header.tpl
  76. +13 −0 src/test/resources/groovyMarkupTemplates/index.tpl
  77. +28 −0 src/test/resources/groovyMarkupTemplates/layout/main.tpl
  78. +9 −0 src/test/resources/groovyMarkupTemplates/menu.tpl
  79. +22 −0 src/test/resources/groovyMarkupTemplates/page.tpl
  80. +20 −0 src/test/resources/groovyMarkupTemplates/post.tpl
  81. +12 −0 src/test/resources/groovyMarkupTemplates/sitemap.tpl
  82. +41 −0 src/test/resources/groovyMarkupTemplates/tags.tpl
  83. +9 −0 src/test/resources/groovyTemplates/allcontent.gsp
  84. +15 −2 src/test/resources/groovyTemplates/tags.gsp
  85. +15 −0 src/test/resources/jadeTemplates/archive.jade
  86. +17 −0 src/test/resources/jadeTemplates/feed.jade
  87. +11 −0 src/test/resources/jadeTemplates/footer.jade
  88. +59 −0 src/test/resources/jadeTemplates/header.jade
  89. +13 −0 src/test/resources/jadeTemplates/index.jade
  90. +20 −0 src/test/resources/jadeTemplates/layout.jade
  91. +12 −0 src/test/resources/jadeTemplates/page.jade
  92. +13 −0 src/test/resources/jadeTemplates/post.jade
  93. +5 −0 src/test/resources/jadeTemplates/sitemap.jade
  94. +15 −0 src/test/resources/jadeTemplates/tags.jade
  95. +29 −0 src/test/resources/pebbleTemplates/archive.pebble
  96. +54 −0 src/test/resources/pebbleTemplates/base.pebble
  97. +25 −0 src/test/resources/pebbleTemplates/feed.pebble
  98. +6 −0 src/test/resources/pebbleTemplates/footer.pebble
  99. +3 −3 src/test/resources/{templates/header.ftl → pebbleTemplates/header.pebble}
  100. +17 −0 src/test/resources/pebbleTemplates/index.pebble
  101. +33 −0 src/test/resources/pebbleTemplates/menu.pebble
  102. +18 −0 src/test/resources/pebbleTemplates/page.pebble
  103. +18 −0 src/test/resources/pebbleTemplates/post.pebble
  104. +9 −0 src/test/resources/pebbleTemplates/sitemap.pebble
  105. +26 −0 src/test/resources/pebbleTemplates/tags.pebble
  106. +0 −7 src/test/resources/templates/allcontent.ftl
  107. +0 −31 src/test/resources/templates/archive.ftl
  108. +0 −13 src/test/resources/templates/footer.ftl
  109. +0 −24 src/test/resources/templates/index.ftl
  110. +0 −18 src/test/resources/templates/page.ftl
  111. +0 −18 src/test/resources/templates/post.ftl
  112. +0 −31 src/test/resources/templates/tags.ftl
  113. +7 −0 src/test/resources/thymeleafTemplates/allcontent.thyme
  114. +11 −1 src/test/resources/thymeleafTemplates/tags.thyme
701 pom.xml

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -0,0 +1,19 @@
<assembly>
<id>base</id>
<includeBaseDirectory>false</includeBaseDirectory>
<!-- Generates a zip package containing the needed files -->
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/example-project-jade/</directory>
<outputDirectory>/</outputDirectory>
<excludes>
<exclude>.git</exclude>
<exclude>README.md</exclude>
<exclude>LICENSE</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
@@ -0,0 +1,19 @@
<assembly>
<id>base</id>
<includeBaseDirectory>false</includeBaseDirectory>
<!-- Generates a zip package containing the needed files -->
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/example-project-pebble/</directory>
<outputDirectory>/</outputDirectory>
<excludes>
<exclude>.git</exclude>
<exclude>README.md</exclude>
<exclude>LICENSE</exclude>
</excludes>
</fileSet>
</fileSets>
</assembly>
@@ -1,13 +1,13 @@
package org.jbake.app;
import java.io.File;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
/**
* Provides Configuration related functions.
*
@@ -21,7 +21,6 @@
*
*/
public static interface Keys {
/**
* Output filename for archive file, is only used when {@link #RENDER_ARCHIVE} is true
*/
@@ -52,6 +51,11 @@
*/
static final String ASSET_FOLDER = "asset.folder";
/**
* URI prefix for content that should be given extensionless output URI's
*/
static final String URI_NO_EXTENSION = "uri.noExtension";
/**
* Flag indicating if hidden asset resources should be ignored
*/
@@ -182,6 +186,15 @@
*/
static final String VERSION = "version";
/**
* Flag indicating if there should be pagination when rendering index
*/
static final String PAGINATE_INDEX = "index.paginate";
/**
* How many posts per page on index
*/
static final String POSTS_PER_PAGE = "index.posts_per_page";
}
private final static Logger LOGGER = LoggerFactory.getLogger(ConfigUtil.class);
@@ -33,7 +33,10 @@
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.jbake.model.DocumentTypes;
@@ -42,7 +45,18 @@
* @author jdlee
*/
public class ContentStore {
public static final String PUBLISHED_DATE = "published_date";
public static final String TAG_POSTS = "tag_posts";
public static final String ALLTAGS = "alltags";
public static final String ALL_CONTENT = "all_content";
public static final String PUBLISHED_CONTENT = "published_content";
public static final String PUBLISHED_PAGES = "published_pages";
public static final String PUBLISHED_POSTS = "published_posts";
public static final String DATABASE = "db";
private ODatabaseDocumentTx db;
private long start = -1;
private long limit = -1;
public ContentStore(final String type, String name) {
db = new ODatabaseDocumentTx(type + ":" + name);
@@ -57,6 +71,27 @@ public ContentStore(final String type, String name) {
}
}
public long getStart() {
return start;
}
public void setStart(int start) {
this.start = start;
}
public long getLimit() {
return limit;
}
public void setLimit(int limit) {
this.limit = limit;
}
public void resetPagination() {
this.start = -1;
this.limit = -1;
}
public final void updateSchema() {
OSchema schema = db.getMetadata().getSchema();
for (String docType : DocumentTypes.getDocumentTypes()) {
@@ -74,6 +109,7 @@ public final void updateSchema() {
public void close() {
db.close();
DBUtil.closeDataStore();
}
public void drop() {
@@ -102,11 +138,19 @@ public long countClass(String iClassName) {
}
public List<ODocument> getPublishedContent(String docType) {
return query("select * from " + docType + " where status='published' order by date desc");
String query = "select * from " + docType + " where status='published'";
if ((start >= 0) && (limit > -1)) {
query += " SKIP " + start + " LIMIT " + limit;
}
return query(query + " order by date desc");
}
public List<ODocument> getAllContent(String docType) {
return query("select * from " + docType + " order by date desc");
String query = "select * from " + docType;
if ((start >= 0) && (limit > -1)) {
query += " SKIP " + start + " LIMIT " + limit;
}
return query(query + " order by date desc");
}
public List<ODocument> getAllTagsFromPublishedPosts() {
@@ -153,6 +197,16 @@ private void executeCommand(String query, Object... args) {
db.command(new OCommandSQL(query)).execute(args);
}
public Set<String> getTags() {
List<ODocument> docs = this.getAllTagsFromPublishedPosts();
Set<String> result = new HashSet<String>();
for (ODocument document : docs) {
String[] tags = DBUtil.toStringArray(document.field("tags"));
Collections.addAll(result, tags);
}
return result;
}
private static void createDocType(final OSchema schema, final String doctype) {
OClass page = schema.createClass(doctype);
page.createProperty("sha1", OType.STRING).setNotNull(true);
@@ -1,15 +1,14 @@
package org.jbake.app;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import org.apache.commons.configuration.CompositeConfiguration;
import org.jbake.app.ConfigUtil.Keys;
import org.jbake.app.Crawler.Attributes.Status;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.io.File.separator;
import java.io.File;
import java.util.Arrays;
@@ -20,14 +19,47 @@
import java.util.Map;
import java.util.Set;
import static java.io.File.separator;
import org.apache.commons.io.FilenameUtils;
import org.jbake.model.DocumentStatus;
import org.jbake.model.DocumentTypes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
/**
* Crawls a file system looking for content.
*
* @author Jonathan Bullock <jonbullock@gmail.com>
*/
public class Crawler {
public static interface Attributes {
/**
* Possible values of the {@link Attributes#STATUS} property
* @author ndx
*
*/
public static interface Status {
static final String PUBLISHED_DATE = "published-date";
static final String PUBLISHED = "published";
static final String DRAFT = "draft";
}
static final String CACHED = "cached";
static final String DATE = "date";
static final String STATUS = "status";
static final String TYPE = "type";
static final String URI = "uri";
static final String FILE = "file";
static final String TAGS = "tags";
static final String TAG = "tag";
static final String RENDERED = "rendered";
static final String SHA1 = "sha1";
static final String ROOTPATH = "rootpath";
}
private static final Logger LOGGER = LoggerFactory.getLogger(Crawler.class);
private CompositeConfiguration config;
@@ -105,7 +137,13 @@ private String buildHash(final File sourceFile) {
private String buildURI(final File sourceFile) {
String uri = FileUtil.asPath(sourceFile.getPath()).replace(FileUtil.asPath( contentPath), "");
// strip off leading / to enable generating non-root based sites
String noExtensionUrlFolder = config.getString(Keys.URI_NO_EXTENSION);
if (!noExtensionUrlFolder.equals("false") && uri.startsWith(noExtensionUrlFolder)) {
uri = "/" + FilenameUtils.getPath(uri) + FilenameUtils.getBaseName(uri) + "/index" + config.getString(Keys.OUTPUT_EXTENSION);
} else {
uri = uri.substring(0, uri.lastIndexOf(".")) + config.getString(Keys.OUTPUT_EXTENSION);
}
// strip off leading / to enable generating non-root based sites
if (uri.startsWith("/")) {
uri = uri.substring(1, uri.length());
}
@@ -115,29 +153,35 @@ private String buildURI(final File sourceFile) {
private void crawlSourceFile(final File sourceFile, final String sha1, final String uri) {
Map<String, Object> fileContents = parser.processFile(sourceFile);
if (fileContents != null) {
fileContents.put("rootpath", getPathToRoot(sourceFile));
fileContents.put("sha1", sha1);
fileContents.put("rendered", false);
if (fileContents.get("tags") != null) {
fileContents.put(Attributes.ROOTPATH, getPathToRoot(sourceFile));
fileContents.put(Attributes.SHA1, sha1);
fileContents.put(Attributes.RENDERED, false);
if (fileContents.get(Attributes.TAGS) != null) {
// store them as a String[]
String[] tags = (String[]) fileContents.get("tags");
fileContents.put("tags", tags);
String[] tags = (String[]) fileContents.get(Attributes.TAGS);
fileContents.put(Attributes.TAGS, tags);
}
fileContents.put("file", sourceFile.getPath());
fileContents.put("uri", uri.substring(0, uri.lastIndexOf(".")) + FileUtil.findExtension(config, fileContents.get("type").toString()));
String documentType = (String) fileContents.get("type");
if (fileContents.get("status").equals("published-date")) {
if (fileContents.get("date") != null && (fileContents.get("date") instanceof Date)) {
if (new Date().after((Date) fileContents.get("date"))) {
fileContents.put("status", "published");
fileContents.put(Attributes.FILE, sourceFile.getPath());
fileContents.put(Attributes.URI, uri);
String documentType = (String) fileContents.get(Attributes.TYPE);
if (fileContents.get(Attributes.STATUS).equals(Status.PUBLISHED_DATE)) {
if (fileContents.get(Attributes.DATE) != null && (fileContents.get(Attributes.DATE) instanceof Date)) {
if (new Date().after((Date) fileContents.get(Attributes.DATE))) {
fileContents.put(Attributes.STATUS, Status.PUBLISHED);
}
}
}
if (!config.getString(Keys.URI_NO_EXTENSION).equals("false")) {
fileContents.put("noExtensionUri", uri.replace("/index.html", "/"));
}
ODocument doc = new ODocument(documentType);
doc.fields(fileContents);
boolean cached = fileContents.get("cached") != null ? Boolean.valueOf((String)fileContents.get("cached")):true;
doc.field("cached", cached);
boolean cached = fileContents.get(Attributes.CACHED) != null ? Boolean.valueOf((String)fileContents.get(Attributes.CACHED)):true;
doc.field(Attributes.CACHED, cached);
doc.save();
} else {
LOGGER.warn("{} has an invalid header, it has been ignored!", sourceFile);
@@ -162,12 +206,12 @@ public String getPathToRoot(File sourceFile) {
public int getDocumentCount(String docType) {
return (int) db.countClass(docType);
}
public Set<String> getTags() {
List<ODocument> query = db.getAllTagsFromPublishedPosts(); //query(new OSQLSynchQuery<ODocument>("select tags from post where status='published'"));
Set<String> result = new HashSet<String>();
for (ODocument document : query) {
String[] tags = DBUtil.toStringArray(document.field("tags"));
String[] tags = DBUtil.toStringArray(document.field(Attributes.TAGS));
Collections.addAll(result, tags);
}
return result;
@@ -177,8 +221,8 @@ private DocumentStatus findDocumentStatus(String docType, String uri, String sha
List<ODocument> match = db.getDocumentStatus(docType, uri);
if (!match.isEmpty()) {
ODocument entries = match.get(0);
String oldHash = entries.field("sha1");
if (!(oldHash.equals(sha1)) || Boolean.FALSE.equals(entries.field("rendered"))) {
String oldHash = entries.field(Attributes.SHA1);
if (!(oldHash.equals(sha1)) || Boolean.FALSE.equals(entries.field(Attributes.RENDERED))) {
return DocumentStatus.UPDATED;
} else {
return DocumentStatus.IDENTICAL;
@@ -4,18 +4,25 @@
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import org.jbake.model.DocumentTypes;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class DBUtil {
public static ContentStore createDB(final String type, String name) {
return new ContentStore(type, name);
}
private static ContentStore contentStore;
public static ContentStore createDataStore(final String type, String name) {
return new ContentStore(type, name);
if (contentStore == null) {
contentStore = new ContentStore(type, name);
}
return contentStore;
}
public static void closeDataStore() {
contentStore = null;
}
public static void updateSchema(final ContentStore db) {
Oops, something went wrong.

0 comments on commit 698cd7a

Please sign in to comment.