Added metadata mode where only the relevant nodes are parsed (faster)

SCDoc.c

@@ -170,7 +170,7 @@ void node_dump(Node *n) {
     _node_dump(n,0,1);
 }
 
-Node * scdoc_parse_file(char *fn, int partial) {
+Node * scdoc_parse_file(char *fn, int mode) {
     FILE *fp;
     Node *n;
 
@@ -181,7 +181,7 @@ Node * scdoc_parse_file(char *fn, int partial) {
     }
     scdoc_current_file = fn;
     scdocrestart(fp);
-    n = scdoc_parse_run(partial);
+    n = scdoc_parse_run(mode);
     if(n) {
         node_fixup_tree(n);
     }

SCDoc.h

@@ -1,6 +1,10 @@
 #ifndef SCDOC_H
 #define SCDOC_H
 
+#define SCDOC_PARSE_FULL 0
+#define SCDOC_PARSE_PARTIAL 1
+#define SCDOC_PARSE_METADATA 2
+
 typedef struct Node {
     const char * id;
     int n_childs;
@@ -16,7 +20,7 @@ Node * node_add_child(Node *n, Node *child);
 Node * node_create(const char *id);
 void node_free_tree(Node *n);
 
-Node * scdoc_parse_file(char *fn, int partial);
+Node * scdoc_parse_file(char *fn, int mode);
 void node_dump(Node *n);
 
 extern char * scdoc_current_file;

SCDoc.l

@@ -10,11 +10,16 @@ int scdoc_start_token = 0;
 
 %}
 
-%option noyywrap never-interactive nounistd
-%option yylineno
+%option noyywrap never-interactive nounistd yylineno
+
 %x verbatim
 %x verbatim2
 
+%x metadata
+%x eat
+%x eat2
+%x eat3
+
 %%
 
 %{
@@ -23,6 +28,7 @@ if (scdoc_start_token) {
     int t = scdoc_start_token;
     scdoc_start_token = 0;
 //    colnum = 1;
+    if(t==START_METADATA) BEGIN(eat);
     return t;
 }
 
@@ -95,11 +101,49 @@ if (scdoc_start_token) {
 [ \r\t]+                scdoclval.str = strdup(" "); return TEXT;
 
 [a-zA-Z]+:\/\/[^ \t\n:,]+       scdoclval.str = strdup(scdoctext); return URL;
+
 [a-zA-Z]+                       |
-<*>[.,!?(){}\[\]'"0-9]+         |
+<INITIAL,verbatim,verbatim2>[.!?(){}\[\]'"0-9]+         |
 <verbatim,verbatim2>[^:\\\t\n ]+ |
 [^:\\\n]                        |
-<*>.                            scdoclval.str = strdup(scdoctext); return TEXT;
-
+<INITIAL,verbatim,verbatim2>.    scdoclval.str = strdup(scdoctext); return TEXT;
+
+<metadata,eat>(?i:[ \t]*class::[ \t]*)         BEGIN(metadata); return CLASS;
+<metadata,eat>(?i:[ \t]*title::[ \t]*)         BEGIN(metadata); return TITLE;
+<metadata,eat>(?i:[ \t]*summary::[ \t]*)       BEGIN(metadata); return SUMMARY;
+<metadata,eat>(?i:[ \t]*related::[ \t]*)       BEGIN(metadata); return RELATED;
+<metadata,eat>(?i:[ \t]*categories::[ \t]*)    BEGIN(metadata); return CATEGORIES;
+<metadata,eat>(?i:[ \t]*redirect::[ \t]*)      BEGIN(metadata); return REDIRECT;
+<metadata,eat>(?i:[ \t]*classtree::[ \t]*)     BEGIN(metadata); return CLASSTREE;
+<metadata,eat>(?i:[ \t]*keyword::[ \t]*)       BEGIN(metadata); return KEYWORD;
+<metadata,eat>(?i:[ \t]*private::[ \t]*)       BEGIN(metadata); return PRIVATE;
+<metadata,eat>(?i:[ \t]*section::[ \t]*)       BEGIN(metadata); return SECTION;
+<metadata,eat>(?i:[ \t]*copymethod::[ \t]*)    BEGIN(metadata); return COPYMETHOD;
+<metadata,eat>(?i:[ \t]*method::[ \t]*)        BEGIN(metadata); return METHOD;
+<metadata,eat>(?i:[ \t]*description::[ \t\n]*)   BEGIN(eat); return DESCRIPTION;
+<metadata,eat>(?i:[ \t]*classmethods::[ \t\n]*)  BEGIN(eat); return CLASSMETHODS;
+<metadata,eat>(?i:[ \t]*instancemethods::[ \t\n]*) BEGIN(eat); return INSTANCEMETHODS;
+<metadata,eat>(?i:[ \t]*examples::[ \t\n]*)      BEGIN(eat); return EXAMPLES;
+<metadata>\n    BEGIN(eat); return NEWLINE;
+<metadata>[ \t]*,[ \t]*           scdoclval.str = strdup(scdoctext); return COMMA;
+<metadata>[a-zA-Z]+ |
+<metadata>[0-9]+    |
+<metadata>[^:\n,]+     |
+<metadata>.         scdoclval.str = strdup(scdoctext); return TEXT;
+<eat>(?i:link::[ \t]*)           BEGIN(eat2);
+<eat>(?i:anchor::[ \t]*)         BEGIN(eat2);
+<eat>(?i:image::[ \t]*)          BEGIN(eat2);
+<eat>(?i:soft::[ \t\n]*)         BEGIN(eat2);
+<eat>(?i:strong::[ \t\n]*)       BEGIN(eat2);
+<eat>(?i:emphasis::[ \t\n]*)     BEGIN(eat2);
+<eat>(?i:code::[ \t]*)           BEGIN(eat2);
+<eat>(?i:teletype::[ \t]*)       BEGIN(eat2);
+<eat>(?i:math::[ \t]*)           BEGIN(eat2);
+<eat>(?i:[ \t]*code::[ \t]*\n+)      BEGIN(eat3);
+<eat>(?i:[ \t]*teletype::[ \t]*\n+)  BEGIN(eat3);
+<eat>(?i:[ \t]*math::[ \t]*\n+)      BEGIN(eat3);
+<eat2>[ \t\n]*::            BEGIN(eat);
+<eat3>\n[ \t\n]*::         BEGIN(eat);
+<eat,eat2,eat3>.|\n          /* empty */
 %%
 

SCDoc.y

@@ -14,6 +14,8 @@ extern char *scdoctext;
 extern int scdoc_start_token;
 //extern struct YYLTYPE scdoclloc;
 
+//int scdoc_metadata_mode;
+
 static const char * method_type = NULL;
 
 static Node * topnode;
@@ -58,7 +60,7 @@ void scdocerror(const char *str);
 %type <node> prose proseelem blockA blockB commalist
 %type <node> deflistbody deflistrow defterms
 
-%token START_FULL START_PARTIAL
+%token START_FULL START_PARTIAL START_METADATA
 
 %start start
 
@@ -86,6 +88,12 @@ document: START_FULL dochead optsections
     {
         $$ = node_make_take_children("BODY",NULL,$2);
     }
+       | START_METADATA dochead optsections
+    {
+        $$ = node_create("DOCUMENT");
+        node_add_child($$, $2);
+        node_add_child($$, $3);
+    }
 ;
 
 dochead: dochead headline { $$ = node_add_child($1,$2); }
@@ -315,8 +323,20 @@ commalist: commalist COMMA nocommawords { free($2); $$ = node_add_child($1,node_
 
 %%
 
-Node * scdoc_parse_run(int partial) {
-    scdoc_start_token = partial? START_PARTIAL : START_FULL;
+Node * scdoc_parse_run(int mode) {
+    int modes[] = {START_FULL, START_PARTIAL, START_METADATA};
+    if(mode<0 || mode>=sizeof(modes)) {
+        fprintf(stderr,"scdoc_parse_run(): unknown mode: %d\n",mode);
+    }
+    scdoc_start_token = modes[mode];
+/*    scdoc_start_token = START_FULL;
+    scdoc_metadata_mode = 0;
+    if(mode==SCDOC_PARSE_PARTIAL) {
+        scdoc_start_token = START_PARTIAL;
+    } else
+    if(mode==SCDOC_PARSE_METADATA) {
+        scdoc_metadata_mode = 1;
+    }*/
     topnode = NULL;
     method_type = "METHOD";
     if(scdocparse()!=0) {

main.c

@@ -9,9 +9,12 @@ int main(int argc, char **argv)
     if(argc>1) {
         Node *n;
         if(argc>2 && strcmp(argv[1],"--partial")==0)
-            n = scdoc_parse_file(argv[2], 1);
+            n = scdoc_parse_file(argv[2], SCDOC_PARSE_PARTIAL);
         else
-            n = scdoc_parse_file(argv[1], 0);
+        if(argc>2 && strcmp(argv[1],"--metadata")==0)
+            n = scdoc_parse_file(argv[2], SCDOC_PARSE_METADATA);
+        else
+            n = scdoc_parse_file(argv[1], SCDOC_PARSE_FULL);
         if(n) {
             node_dump(n);
             node_free_tree(n);

notes.txt

@@ -93,6 +93,7 @@ real plans:
             title, summary, categories, related, redirect
         and the nodes needed for metadata:
             method, cmethod, imethod, private, keyword, copymethod
+        Use a SCDocMapEntry for this?
 
         note: the current implementation also adds undocumented methods when
         it's a class. we need to do this on the lang-side.
@@ -105,5 +106,21 @@ real plans:
 
         note that we insert undocumented methods already in the document tree,
         while other introspection data is inserted at rendertime.
+
+4. move inherit-mets from JS to renderer.
+   then, do we really need the subclasses/superclasses in docmap.js?
+   yes, maybe for method search results ("class A inherited by B, etc..")
+
+5. If metadata parsing is fast enough, do it fully at each startup instead of caching it.
+   yes, the whole SC helpsource tree takes 1.5 secs on my machine
+
+6. note that some docs need updating even if src not-newer-than dest:
+   - class changed (check mtime of Class.filenameSymbol?)
+     then we need to store the old mtime's on disk..
+   - metadata changed (for link:: titles)
+     not sure what to do about that.. actually it must be solved by a dependency system.
+     docs that links to other docs has those as dependencies.
+   - class hier changed (for classtree:: tag)
+     this means we need to know if a doc uses classtree::, could be marked in the metadata.
 ...