Skip to content
This repository
Newer
Older
100755 529 lines (465 sloc) 19.264 kb
847c9393 »
2011-09-25 added version of newlispdoc that allows external syntax-hilite
1 #!/usr/bin/env newlisp
2
3 ;; @module newlispdoc
176a8685 »
2011-10-03 spelling
4 ;; @description Generates documentation and highlighted source from newLISP source files.
847c9393 »
2011-09-25 added version of newlispdoc that allows external syntax-hilite
5 ;; @version 1.3 - handle remote files specified in URLs in a url-file
6 ;; @version 1.4 - title on page changed to index or module name and new index module option
7 ;; @version 1.5 - removed font restrictions on h1,h2,h3,h4 and added hr as a legal tag
8 ;; @version 1.6 - don't allow 3 semicolons at the beginning of a comment line
176a8685 »
2011-10-03 spelling
9 ;; @version 1.7 - 1.6 did not sense lines with semicolon only as paragraph separators
847c9393 »
2011-09-25 added version of newlispdoc that allows external syntax-hilite
10 ;; @version 1.8 - fixed problem of "true" appearing before multiple syntax statements
11 ;; @version 1.9 - write-line is now version sensitive for 10.0
12 ;; @version 2.0 - adds handling of < ?...> XML tag
13 ;; @version 2.1 - generate more compliant HTML 4.01 for doc and syntax highlighting
176a8685 »
2011-10-03 spelling
14 ;; but special arithmetic op characters in functionnames still cause
847c9393 »
2011-09-25 added version of newlispdoc that allows external syntax-hilite
15 ;; non-compliant HTML (this occurs only in modules/gmp.lsp)
16 ;; @version 2.2 - fix issue with paragraph spacing and module text font from 2.1
17 ;; @version 2.3 - handle scientific notation with e
18 ;; @version 2.4 - improved keyword regex for syntax highlighting
19 ;; @version 2.5 - scientific notation with E
20 ;; @version 2.6 - allow custom tags e.g: @MyTag the descriptive text (may contain @link)
176a8685 »
2011-10-03 spelling
21 ;; @version 2.7 - link rag did not work at beginning of line
847c9393 »
2011-09-25 added version of newlispdoc that allows external syntax-hilite
22 ;; @version 2.8 - support for newLISPdoc tag color
23 ;; @version 2.9 - correctly highlight & and ^ keywords
24 ;; @version 3.0 - when -d is specified, add download link
25 ;; @version 3.1 - changed <tt>write-buffer</tt> to shorter <tt>write</tt>, <tt>name</tt> to <tt>term</tt>
26 ;; @version 3.2 - formatting when view source, new stylesheet newlispdoc.css
27 ;; @version 3.3 - forked, added option to subcontract syntax highlighting out to external script (cormullion)
28 ;;
29 ;; @author Lutz Mueller, 2006-2010, cormullion 2011
30 ;; @location http://newlisp.org
31 ;;
32 ;; <h3>Usage:</h3>
33 ;; The <tt>-s</tt> and <tt>-s</tt> switches can be used to generate highlighted source files,
34 ;; links and and download links
35 ;; <pre>
36 ;; newlispdoc afile.lsp bfile.lsp
37 ;; newlispdoc -s afile.lsp bfile.lsp
38 ;; newlispdoc -s *.lsp
39 ;; newlispdoc -s -d *.lsp
40 ;; newlispdoc -e extformatter.lsp *.lsp
41 ;; newlispdoc -e extformatter.lsp -d *.lsp
42 ;; <br>
43 ;; ; or on Win32
44 ;; newlisp newlispdoc afile.lsp bfile.lsp
45 ;; newlisp newlispdoc -s afile.lsp bfile.lsp
46 ;; newlisp newlispdoc -s *.lsp
47 ;; newlisp newlispdoc -s -d *.lsp
48 ;; newlispdoc -e extformatter.lsp *.lsp
49 ;; </pre>
50 ;; newlispdoc can take a file of URLs and generate documentation and syntax
51 ;; highlighted source from remote files:
52 ;; <pre>
53 ;; newlispdoc -url file-with-urls.txt 10000
54 ;; newlispdoc -s -url file-with-urls.txt 10000
55 ;; <br>
56 ;; ; or on Win32
57 ;; newlisp newlispdoc -url file-with-urls.txt 10000
58 ;; newlisp newlispdoc -s -url file-with-urls.txt 10000
59 ;; </pre>
60 ;; file-with-urls.txt contains one URL per line in the file. A URLstarts either
61 ;; with http:// or file:// - This allows mixing remote and local files. An optional
62 ;; timeout of 10 seconds is specified after the url file name. If no timeout is specified
63 ;; newlispdoc assumes 5 seconds.
64 ;;
65 ;; Execute from within the same directory of the source files when
66 ;; no url file is given.
67 ;;
68 ;; For each file a file with the same name and extension '.html' is generated
69 ;; and written to the current directory. A file 'index.htm' is written as
70 ;; an index for all other files generated. If the '-s' switch is specified,
71 ;; a file with the extension '.src.html' is generated for each file.
72 ;;
73 ;; If the '-e' switch is given, the syntax highlighting, and the resulting .src.html file, will be generated
74 ;; by the named newLISP script, rather than by the built-in version. This allows
75 ;; you to choose alternative syntax highlighting programs. See the 'external-syntax-highlight' function below.
76 ;;
77 ;; Please read @link http://newlisp.org/newLISPdoc.html newLISPdoc.html to learn
78 ;; about tagging of newLISP source code for newlispdoc.
79 ;;
80
81 ; make compatible with older versions of newLISP
82 (when (< (sys-info -2) 10111)
83 (constant (global 'write) write-buffer)
84 (constant (global 'term) name))
85
86 ; get list of files from command line
87 (set 'files (2 (main-args)))
88
89 (if (empty? files)
90 (begin
91 (println "USAGE: newlispdoc [-s] [-d] <file-names>")
92 (println "USAGE: newlispdoc [-s] [-d] -url <file-with-urls>")
93 (println "USAGE: newlispdoc [-e file.lsp] [-d] -url <file-with-urls>")
94 (exit -1)))
95
96 (set 'prolog1
97 [text]<!DOCTYPE HTML PUBLIC "HTML 4.01 Transitional">
98 <html>
99 <head>
100 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
101 <title>%s</title>
102 [/text])
103
104 (set 'stylesheet
105 [text]
106 <link rel="stylesheet" type="text/css" href="newlispdoc.css" />
107 </head>
108 [/text])
109
110 (set 'prolog2
111 [text]
112 <body style="margin: 20px;" text="#111111" bgcolor="#FFFFFF"
113 link="#376590" vlink="#551A8B" alink="#ffAA28">
114 <blockquote>
115 <center><h1>%s</h1></center>
116 [/text])
117
118 (set 'prolog3 {<p><a href="index.html">Module index</a></p>})
119
120 (set 'epilog [text]
121 <br/><br/><center>- &part; -</center><br/>
122 <center><font face='Arial' size='-2' color='#444444'>
123 generated with <a href="http://newlisp.org">newLISP</a>&nbsp;
124 and <a href="http://newlisp.org/newLISPdoc.html">newLISPdoc</a>
125 </font></center>
126 </blockquote>
127 </body>
128 </html>
129 [/text])
130
131 ; ---------- routines for generating syntax-highlighted file <file-name>.src.html ------------------
132
133 (define keyword-color "#0000AA") ; newLISP keywords
134 (define tag-color "#308080") ; newLISPdoc tags
135 (define string-color "#008800") ; single line quoted and braced strings
136 (define long-string-color "#008800") ; multiline for [text], [/text] tags
137 (define paren-color "#AA0000") ; parenthesis
138 (define comment-color "#555555") ; comments
139 (define number-color "#665500") ; numbers
140
141 (define function-name-color "#000088") ; not implemented yet for func in (define (func x y z) ...)
142
143 (set 'keywords (map term (filter (fn (x) (primitive? (eval x))) (sort (symbols) > ))))
144 (push "nil" keywords)
145 (push "true" keywords)
146 (set 'keyword-regex (join keywords "|"))
147 (replace "&" keyword-regex "&amp&")
148 (replace "?" keyword-regex "\\?")
149 (replace "^" keyword-regex "\\^")
150 (replace "$" keyword-regex "\\$")
151 (replace "!" keyword-regex "\\!")
152 (replace "+" keyword-regex "\\+")
153 (replace "*" keyword-regex "\\*")
154 (replace "||" keyword-regex "|\\|")
155 (set 'keyword-regex (append {(\G|\s+|\(|\))(} keyword-regex {)(\s+|\(|\))}))
156
157 (define (clean-comment str)
158 (replace {<font color='#......'>} str "" 0)
159 (replace {</font>} str "")
160 (replace {[text]} str "&#091&text]")
161 (replace {[/text]} str "&#091&/text]")
162 )
163
164 (define (format-quoted-string str)
165 (replace {<font color='#......'>} str "" 0)
166 (replace {</font>} str "")
167 (replace ";" str "&#059&")
168 (replace "{" str "&#123&")
169 (replace "}" str "&#125&")
170 (replace {\} str "&#092&")
171 (replace {[text]} str "&#091&text]")
172 (replace {[/text]} str "&#091&/text]")
173 (format {<font color='%s'>%s</font>} string-color str)
174 )
175
176 (define (format-braced-string str)
177 (replace {<font color='#......'>} str "" 0)
178 (replace {</font>} str "")
179 (replace ";" str "&#059&")
180 (replace {"} str "&#034&")
181 (replace {[text]} str "&#091&text]")
182 (replace {[/text]} str "&#091&/text]")
183 (format {<font color='%s'>%s</font>} string-color str)
184 )
185
186 (define (format-tagged-string str)
187 (replace {<font color='#......'>} str "" 0)
188 (replace {</font>} str "")
189 (replace ";" str "&#059&")
190 (format {<font color='%s'>%s</font>} string-color str)
191 )
192
193 (define (write-syntax-highlight title src-file outputfile)
194 ; replace special HTML characters
195 (replace "\r\n" src-file "\n")
196 (replace "&" src-file "&amp&")
197 (replace "<(\\w)" src-file (append "&lt&" $1) 0)
198 (replace "(\\w)>" src-file (append $1 "&gt&") 0)
199 (replace "/>" src-file "/&gt&" 0)
200 (replace "</" src-file "&lt&/" 0)
201 (replace "<!" src-file "&lt&!" 0)
202 (replace "<\\?" src-file "&lt&?" 0)
203 ; replace escaped quotes
204 (replace "\092\034" src-file "&#092&&#034&")
205
206 ; color keywords
207 (replace keyword-regex src-file
208 (format {%s<font color='%s'>%s</font>%s} $1 keyword-color $2 $3) 0)
209
210 ; color numbers
211 (replace
212 ; <-- lead --><---- hex ---->|<- oct ->|<------- decimal ----- and ----- scientific -->
213 {(\s+|\(|\))(0x[0-9a-fA-F]+|[+-]?0\d+|([+-]?(0|[1-9]\d*)(\.\d*)?|\.\d+)([eE][+-]?\d+)?)} src-file
214 (format {%s<font color='%s'>%s</font>} $1 number-color $2) 0)
215
216 ; color parens
217 (replace "(" src-file (format {<font color='%s'>(</font>} paren-color))
218 (replace ")" src-file (format {<font color='%s'>)</font>} paren-color))
219
220 ; color braced strings
221 (replace "{.*?}" src-file (format-braced-string $0) 0) ; no multiline string
222 ; color quoted strings
223 (replace {".*?"} src-file (format-quoted-string $0) 0) ; no multiline strings
224
225 ; color ; comments
226 (replace ";.*" src-file (clean-comment $0) 0)
227 (replace ";.*" src-file (format {<font color='%s'>%s</font>} comment-color $0) 0)
228 (replace "(;;.*)(@.*)(\\s.*\n)" src-file
229 (append $1 (format {<font color='%s'>%s</font>} tag-color $2) $3) 512)
230
231 ; color # comments
232 (set 'buff "")
233 (dolist (lne (parse src-file "\n"))
234 (replace "^\\s*#.*" lne (clean-comment $0) 0)
235 (replace "^\\s*#.*" lne (format {<font color='%s'>%s</font>} comment-color $0) 0)
236 (if (< (sys-info -2) 9909)
237 (write-line lne buff)
238 (write-line buff lne)))
239
240 (set 'src-file buff)
241
242 ; color tagged strings
243 (replace {\[text\].*?\[/text\]} src-file
244 (format-tagged-string $0) 4) ; handles multiline strings
245
246 ; xlate back special characters
247 (replace "&amp&" src-file "&amp;") ; ampersand
248 (replace "&lt&" src-file "&lt;") ; less
249 (replace "&gt&" src-file "&gt;") ; greater
250 (replace {&#034&} src-file "&#034;") ; quotes
251 (replace {&#059&} src-file "&#059;") ; semicolon
252 (replace {&#123&} src-file "&#123;") ; left curly brace
253 (replace {&#125&} src-file "&#125;") ; right curly brace
254 (replace {&#091&} src-file "&#091;") ; left bracket
255 (replace {&#092&} src-file "&#092;") ; back slash
256
257 ; add pre and post tags
258 (write-file (string outputfile ".src.html")
259 (append
260 {<!DOCTYPE HTML PUBLIC "4.01 Transitional">}
261 "<html><title>" title "</title><body><pre>\n" src-file "\n</pre>"
262 {<center><font face='Arial' size='-3' color='#666666'>}
263 {syntax highlighting with <a href="http://newlisp.org">newLISP</a>&nbsp;}
264 {and <a href="http://newlisp.org/newLISPdoc.html">newLISPdoc</a>}
265 {</font></center></body></html>}))
266 )
267
268 (define (external-syntax-highlight script-name title src-file outputfile)
269 ; the 'script-name' newLISP file is loaded. It should supply a function taking two arguments,
270 ; source-text and title, and returning a string of the highlighted source:
271 ; eg: (define (syntax-highlight source-text title) (string title source-text))
272 (load script-name)
273 (set 'highlighted-source (syntax-highlight src-file title))
274 (write-file (string outputfile ".src.html") highlighted-source))
275
276 ;---------------------------------- End syntax highlighting routines ------------------------
277
278 ; get command line switch -s for generating syntax highlighted source
279 (let (pos (find "-s" files))
280 (when pos
281 (pop files pos)
282 (set 'source-link true)))
283
284 ; get command line switch -e for delegating syntax highlighting to external script
285 (let (pos (find "-e" files))
286 (when pos
287 (pop files pos)
288 (set 'syntax-highlighter-script (files pos))
289 (pop files)
290 (println { ... will use '} syntax-highlighter-script {' for generating syntax highlighting})
291 (set 'source-link true 'source-highlight true)))
292
293 ; get command line switch -d for generating a download link
294 (let (pos (find "-d" files))
295 (when pos
296 (pop files pos)
297 (set 'download-link true)))
298
299 ; get command line switch -url for retrieving files via http from remote location
300 (let (pos (find "-url" files))
301 (when pos
302 (pop files pos)
303 (set 'url-file (files pos))
304 (set 'time-out (int (last files) 5000))))
305
306 ; if url-file is specified make a list of all urls
307 (if url-file
308 (set 'files (parse (read-file url-file) "\\s+" 0)))
309
310 (if (= (last files) "") (pop files -1))
311
312 (set 'indexpage "") ; buffer for modules index page
313 (write indexpage (format prolog1 "Index"))
314 (write indexpage stylesheet)
315 (write indexpage (format prolog2 "Index"))
316
317 ; reformat
318
319 (define (protect-html text)
320 (replace "<h1>" text "[h1]")
321 (replace "<h2>" text "[h2]")
322 (replace "<h3>" text "[h3]")
323 (replace "<h4>" text "[h4]")
324 (replace "</h1>" text "[/h1]")
325 (replace "</h2>" text "[/h2]")
326 (replace "</h3>" text "[/h3]")
327 (replace "</h4>" text "[/h4]")
328 (replace "<i>" text "[i]")
329 (replace "</i>" text "[/i]")
330 (replace "<em>" text "[em]")
331 (replace "</em>" text "[/em]")
332 (replace "<b>" text "[b]")
333 (replace "</b>" text "[/b]")
334 (replace "<tt>" text "[tt]")
335 (replace "</tt>" text "[/tt]")
336 (replace "<p>" text "[p]")
337 (replace "</p>" text "[/p]")
338 (replace "<br>" text "[br]")
339 (replace "<br/>" text "[br/]")
340 (replace "<pre>" text "[pre]")
341 (replace "</pre>" text "[/pre]")
342 (replace "<center>" text "[center]")
343 (replace "</center>" text "[/center]")
344 (replace "<li>" text "[li]")
345 (replace "</li>" text "[/li]")
346 (replace "</ul>" text "[/ul]")
347 (replace "<ul>" text "[ul]")
348 (replace "</blockquote>" text "[/blockquote]")
349 (replace "<blockquote>" text "[blockquote]")
350 (replace "<hr>" text "[hr]")
351 (replace "<hr/>" text "[hr/]")
352 )
353
354 (define (unprotect-html text)
355 (replace "[h1]" text "<h1>")
356 (replace "[h2]" text "<h2>")
357 (replace "[h3]" text "<h3>")
358 (replace "[h4]" text "<h4>")
359 (replace "[/h1]" text "</h1>")
360 (replace "[/h2]" text "</h2>")
361 (replace "[/h3]" text "</h3>")
362 (replace "[/h4]" text "</h4>")
363 (replace "[i]" text "<i>")
364 (replace "[/i]" text "</i>")
365 (replace "[em]" text "<em>")
366 (replace "[/em]" text "</em>")
367 (replace "[b]" text "<b>")
368 (replace "[/b]" text "</b>")
369 (replace "[tt]" text "<tt>")
370 (replace "[/tt]" text "</tt>")
371 (replace "[p]" text "<p>")
372 (replace "[/p]" text "</p>")
373 (replace "[br]" text "<br>")
374 (replace "[br/]" text "<br/>")
375 (replace "[pre]" text "<pre>")
376 (replace "[/pre]" text "</pre>")
377 (replace "[center]" text "<center>")
378 (replace "[/center]" text "</center>")
379 (replace "[li]" text "<li>")
380 (replace "[/li]" text "</li>")
381 (replace "[ul]" text "<ul>")
382 (replace "[/ul]" text "</ul>")
383 (replace "[blockquote]" text "<blockquote>")
384 (replace "[/blockquote]" text "</blockquote>")
385 (replace "[hr]" text "<hr>")
386 (replace "[hr/]" text "<hr/>")
387 )
388
389 ; format the example tags
390 (define (format-example text)
391 (replace "<" text "&lt;")
392 (replace ">" text "&gt;")
393 (string "<b>example:</b><blockquote><pre>" (replace ";;" text "") "</pre></blockquote>\n")
394 )
395
396 ; write the module tag link on the index page
397 ; put source link on doc page if -s flag
398 (define (format-module text desc filename , module)
399 (set 'module (string "<br/>\n<h2>Module:&nbsp;" text "</h2>"))
400 (write indexpage (string {<a href="} filename {.html">} module "</a>\n" ))
401 (write indexpage (string "<p>" desc "</p>\n"))
402
403 (let (link "")
404 (if source-link (setq link (string {<a href="} filename {.src.html">source</a>&nbsp;})))
405 (if download-link (write link (string {<a href="} filename {">download</a>})))
406 (string link module)
407 )
408 )
409
410 ; write the function name links on the index page under the module
411 (define (format-func-link func-name file-name)
412 (let (names (if (find ":" func-name) (parse func-name ":") (list "" func-name)))
413 (write indexpage (string {<a href="} file-name
414 {.html#} (names 0) "_" (names 1) {">} (names 1) {</a>&nbsp; &nbsp; }))
415 (string (names 0) "_" (names 1))
416 )
417 )
418
419 ; format the syntax line
420 (define (format-syntax text file-name, tag)
421 (replace "<([^<]*?)>" text (string "<em>" $1 "</em>") 0)
422 (replace {^ *\((.*?) (.*?\))} text (string "(<font color=#CC0000>" $1 "</font> " $2) 0)
423 (replace {^ *\(([^ ]*?)\)} text (string "(<font color=#CC0000>" $1 "</font>)") 0)
424 (string
425 (if (!= old-syntax $1)
426 (begin
427 (set 'old-syntax $1)
428 (set 'tag (format-func-link $1 file-name))
429 (set 'tag (string {<a name="} tag {"></a>}))
430 (string "<br/><br/><center>&sect;</center><br/>\n" tag
431 "<h3><font color=#CC0000>" old-syntax "</font></h3>\n"))
432 "")
433 "<b>syntax: " (trim text) "</b><br/>\n")
434 )
435
436 (define (format-parameter param text)
437 (string "<b>parameter: </b>" (format-text (trim param)) " - " (format-text text) "<br/>\n")
438 )
439
440 (define (format-return text)
441 (string "<p><b>return: </b>" (format-text text) "</p>\n")
442 )
443
444 (define (format-text text)
445 (replace "<([^<]*?)>" text (string "<em>" $1 "</em>") 0)
446 (replace "'([^\\s].*?[^\\s])'" text (string "<tt>" $1 "</tt>") 0)
447 )
448
449 ;---------------------------------- End newlisdoc formatting subroutines ----------------
450 ; MAIN function
451
452 (dolist (fle files)
453 (println " ... processing file " fle)
454 (set 'html "")
455
456 (set 'original (read-file fle time-out))
457
458 (if (not original)
459 (begin
460 (println "Could not read " fle)
461 (exit 1)))
462
463 (if (starts-with original "ERR:")
464 (println "Could not read " fle " " original))
465
466 (set 'outfile (last (parse fle "\\\\|/" 0)))
467
468 (set 'page (parse (replace "\r\n" original "\n") "\n"))
469
470 (set 'page (filter (fn (ln) (or (starts-with ln ";;(?!;)" 0) (= (length (trim ln)) 0))) page))
471 (set 'page (join page "\n"))
472
473 ; link to another index page
474 (if (find ";; *@index ([^ ]*?) ([^ ]*?)\n" page 0)
475 (begin
476 (write indexpage
477 (string {<br /><br /><h2><a href="} $2 {">Index:&nbsp;} $1 "</a></h2>\n"))
478 (if (find ";; *@description (.*?)\n" page 0)
479 (write indexpage (string $1 "<br/>\n")))
480 )
481 )
482
483
484 (if (find ";; *@module " page 0)
485 (begin
486 (replace {;;(.*) @link ([^ ]*?) ([^ ]*?)\s} page (string $1 { <a href="} $2 {">} $3 {</a> }) 0)
487 (replace ";; @example *\n(.*?)\n\\s*\n" page (format-example $1) 4)
488 (set 'page (protect-html page))
489 (set 'desc "")
490 (replace ";; *@description (.*?)\n" page (begin (set 'desc $1) (string "<p>" desc "</p>\n") ) 0)
491 (replace ";; *@module (.*?)\n" page (format-module $1 desc outfile) 0)
492 (set 'module-title $1)
493 (replace ";; *@author (.*?)\n" page (string "<b>Author: </b>" $1 "<br/>\n") 0)
494 (replace ";; *@version (.*?)\n" page (string "<b>Version: </b>" $1 "<br/>\n") 0)
495 (replace ";; *@location (.*?)\n" page
496 (string {<b>Location: </b><a href="} $1 {">} $1 "</a><br/>\n") 0)
497 (replace ";; *@syntax (.*?)\n" page (format-syntax $1 outfile) 0)
498 (replace ";; *@param (.*?) (.*?)\n" page (format-parameter $1 $2) 0)
499 (replace ";; *@return (.*?)\n" page (format-return $1) 0)
500 (replace ";; *@([a-zA-Z_-]*?) (.*?)\n" page (string "<b>" $1 ": </b>" $2 "<br/>\n") 0)
501 (replace ";;\\s*\n" page "<br/><br/>\n" 0)
502 (replace ";;(.*\n)" page (format-text $1) 0)
503 (set 'page (unprotect-html page))
504 (write html (format prolog1 outfile))
505 (write html stylesheet)
506 (write html (format prolog2 outfile))
507 (write html prolog3)
508 (write html page)
509 (write html epilog)
510
511 (write-file (string outfile ".html") html)
512
513 ; write syntax highlighted source using built-in routines
514 (if source-link
515 (write-syntax-highlight module-title original outfile))
516
517 ; or using external script
518 (if (and source-highlight source-link)
519 (external-syntax-highlight syntax-highlighter-script module-title original outfile))
520 )
521 )
522 )
523
524 ; write the modules index page
525 (write indexpage epilog)
526 (write-file "index.html" indexpage)
527
528 (exit)
529 ;eof
Something went wrong with that request. Please try again.