Refactoring of language services to use JSON metadata

README.md

@@ -1,6 +1,6 @@
 # Inform 7
 
-v10.1.0-beta+6V26 'Krypton' (21 June 2022)
+v10.1.0-beta+6V27 'Krypton' (23 June 2022)
 
 ## About Inform 7
 

docs/building-module/2-tkn.html

@@ -121,14 +121,21 @@ <h1><a href="../index.html">
 <span class="plain-syntax">                    </span><span class="identifier-syntax">PUT_TO</span><span class="plain-syntax">(</span><span class="identifier-syntax">current_raw</span><span class="plain-syntax">, </span><span class="identifier-syntax">c</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                }</span>
 <span class="plain-syntax">                </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
-<span class="plain-syntax">             </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">SQUOTED_TOKSTATE:</span>
+<span class="plain-syntax">             </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">SQUOTED_TOKSTATE:</span><span class="plain-syntax"> {</span>
+<span class="plain-syntax">                </span><span class="reserved-syntax">int</span><span class="plain-syntax"> </span><span class="identifier-syntax">ends_here</span><span class="plain-syntax"> = </span><span class="identifier-syntax">FALSE</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                </span><span class="reserved-syntax">if</span><span class="plain-syntax"> (</span><span class="identifier-syntax">c</span><span class="plain-syntax"> == </span><span class="character-syntax">'\''</span><span class="plain-syntax">) {</span>
+<span class="plain-syntax">                    </span><span class="identifier-syntax">ends_here</span><span class="plain-syntax"> = </span><span class="identifier-syntax">TRUE</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">                    </span><span class="reserved-syntax">if</span><span class="plain-syntax"> ((</span><span class="identifier-syntax">Str::len</span><span class="plain-syntax">(</span><span class="identifier-syntax">current_raw</span><span class="plain-syntax">) == </span><span class="constant-syntax">0</span><span class="plain-syntax">) &amp;&amp; (</span><span class="identifier-syntax">Str::get_at</span><span class="plain-syntax">(</span><span class="identifier-syntax">from</span><span class="plain-syntax">, </span><span class="identifier-syntax">pos</span><span class="plain-syntax">+1) == </span><span class="character-syntax">'\''</span><span class="plain-syntax">))</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">ends_here</span><span class="plain-syntax"> = </span><span class="identifier-syntax">FALSE</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">                }</span>
+<span class="plain-syntax">                </span><span class="reserved-syntax">if</span><span class="plain-syntax"> (</span><span class="identifier-syntax">ends_here</span><span class="plain-syntax">) {</span>
 <span class="plain-syntax">                    </span><span class="named-paragraph-container code-font"><a href="2-tkn.html#SP1_3" class="named-paragraph-link"><span class="named-paragraph">Absorb raw material, for sure</span><span class="named-paragraph-number">1.3</span></a></span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                    </span><span class="identifier-syntax">tokeniser_state</span><span class="plain-syntax"> = </span><span class="constant-syntax">NO_TOKSTATE</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                } </span><span class="reserved-syntax">else</span><span class="plain-syntax"> {</span>
 <span class="plain-syntax">                    </span><span class="identifier-syntax">PUT_TO</span><span class="plain-syntax">(</span><span class="identifier-syntax">current_raw</span><span class="plain-syntax">, </span><span class="identifier-syntax">c</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                }</span>
 <span class="plain-syntax">                </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">             }</span>
 <span class="plain-syntax">             </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">COMMENT_TOKSTATE:</span>
 <span class="plain-syntax">                </span><span class="reserved-syntax">if</span><span class="plain-syntax"> (</span><span class="identifier-syntax">c</span><span class="plain-syntax"> == </span><span class="character-syntax">'\n'</span><span class="plain-syntax">) </span><span class="identifier-syntax">tokeniser_state</span><span class="plain-syntax"> = </span><span class="constant-syntax">NO_TOKSTATE</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>

docs/core-module/1-cp2.html

@@ -65,11 +65,12 @@ <h1><a href="../index.html">
 </p>
 
 <pre class="displayed-code all-displayed-code code-font">
-<span class="reserved-syntax">void</span><span class="plain-syntax"> </span><span class="function-syntax">CorePreform::load</span><span class="plain-syntax">(</span><span class="identifier-syntax">inform_language</span><span class="plain-syntax"> *</span><span class="identifier-syntax">L</span><span class="plain-syntax">) {</span>
+<span class="reserved-syntax">int</span><span class="plain-syntax"> </span><span class="function-syntax">CorePreform::load</span><span class="plain-syntax">(</span><span class="identifier-syntax">inform_language</span><span class="plain-syntax"> *</span><span class="identifier-syntax">L</span><span class="plain-syntax">) {</span>
 <span class="plain-syntax">    </span><span class="reserved-syntax">if</span><span class="plain-syntax"> (</span><span class="identifier-syntax">L</span><span class="plain-syntax"> == </span><span class="identifier-syntax">NULL</span><span class="plain-syntax">) </span><span class="identifier-syntax">internal_error</span><span class="plain-syntax">(</span><span class="string-syntax">"can't load preform from null language"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">    </span><span class="identifier-syntax">filename</span><span class="plain-syntax"> *</span><span class="identifier-syntax">F</span><span class="plain-syntax"> = </span><span class="identifier-syntax">Filenames::in</span><span class="plain-syntax">(</span><span class="identifier-syntax">Languages::path_to_bundle</span><span class="plain-syntax">(</span><span class="identifier-syntax">L</span><span class="plain-syntax">), </span><span class="identifier-syntax">I</span><span class="string-syntax">"Syntax.preform"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">    </span><span class="reserved-syntax">int</span><span class="plain-syntax"> </span><span class="identifier-syntax">nonterminals_declared</span><span class="plain-syntax"> = </span><span class="identifier-syntax">LoadPreform::load</span><span class="plain-syntax">(</span><span class="identifier-syntax">F</span><span class="plain-syntax">, </span><span class="identifier-syntax">L</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">    </span><span class="identifier-syntax">LOG</span><span class="plain-syntax">(</span><span class="string-syntax">"%d Preform nonterminals read from %f\n"</span><span class="plain-syntax">, </span><span class="identifier-syntax">nonterminals_declared</span><span class="plain-syntax">, </span><span class="identifier-syntax">F</span><span class="plain-syntax">);</span>
+<span class="plain-syntax">    </span><span class="reserved-syntax">return</span><span class="plain-syntax"> </span><span class="identifier-syntax">nonterminals_declared</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">}</span>
 </pre>
 <p class="commentary firstcommentary"><a id="SP2" class="paragraph-anchor"></a><b>&#167;2. Converting Preform errors to problems. </b>Errors in Preform syntax are generated in the <a href="../words-module/index.html" class="internal">words</a> module, and are

docs/core-module/2-pwst.html

@@ -317,10 +317,12 @@ <h1><a href="../index.html">
 <span class="plain-syntax">                            </span><span class="string-syntax">"has an 'ends here' with nothing having begun"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                        </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                    </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">BadTitleSentence_SYNERROR:</span>
-<span class="plain-syntax">                        </span><span class="identifier-syntax">current_sentence</span><span class="plain-syntax"> = </span><span class="identifier-syntax">CE</span><span class="plain-syntax">-&gt;</span><span class="identifier-syntax">details_node</span><span class="plain-syntax">;</span>
-<span class="plain-syntax">                        </span><span class="identifier-syntax">StandardProblems::sentence_problem</span><span class="plain-syntax">(</span><a href="1-wtc.html#SP6" class="function-link"><span class="function-syntax">Task::syntax_tree</span></a><span class="plain-syntax">(), </span><span class="identifier-syntax">_p_</span><span class="plain-syntax">(</span><span class="identifier-syntax">PM_BadTitleSentence</span><span class="plain-syntax">),</span>
-<span class="plain-syntax">                            </span><span class="string-syntax">"the initial bibliographic sentence can only be a title in double-quotes"</span><span class="plain-syntax">,</span>
-<span class="plain-syntax">                            </span><span class="string-syntax">"possibly followed with 'by' and the name of the author."</span><span class="plain-syntax">);</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">current_sentence</span><span class="plain-syntax"> = </span><span class="identifier-syntax">NULL</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">StandardProblems::unlocated_problem</span><span class="plain-syntax">(</span><a href="1-wtc.html#SP6" class="function-link"><span class="function-syntax">Task::syntax_tree</span></a><span class="plain-syntax">(),</span>
+<span class="plain-syntax">                            </span><span class="identifier-syntax">_p_</span><span class="plain-syntax">(</span><span class="identifier-syntax">PM_BadTitleSentence</span><span class="plain-syntax">),</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"The opening bibliographic sentence can only be a title in "</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"double-quotes, possibly followed with 'by' and the name of "</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"the author."</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                        </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                    </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">UnknownLanguageElement_SYNERROR:</span>
 <span class="plain-syntax">                        </span><span class="identifier-syntax">current_sentence</span><span class="plain-syntax"> = </span><span class="identifier-syntax">CE</span><span class="plain-syntax">-&gt;</span><span class="identifier-syntax">details_node</span><span class="plain-syntax">;</span>
@@ -461,6 +463,16 @@ <h1><a href="../index.html">
 <span class="plain-syntax">                            </span><span class="string-syntax">"was %4.)"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                        </span><span class="identifier-syntax">Problems::issue_problem_end</span><span class="plain-syntax">();</span>
 <span class="plain-syntax">                        </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">                    </span><span class="reserved-syntax">case</span><span class="plain-syntax"> </span><span class="identifier-syntax">UnavailableLOS_SYNERROR:</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">current_sentence</span><span class="plain-syntax"> = </span><span class="identifier-syntax">NULL</span><span class="plain-syntax">;</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">Problems::quote_stream</span><span class="plain-syntax">(1, </span><span class="identifier-syntax">CE</span><span class="plain-syntax">-&gt;</span><span class="identifier-syntax">details</span><span class="plain-syntax">);</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">StandardProblems::handmade_problem</span><span class="plain-syntax">(</span><a href="1-wtc.html#SP6" class="function-link"><span class="function-syntax">Task::syntax_tree</span></a><span class="plain-syntax">(), </span><span class="identifier-syntax">_p_</span><span class="plain-syntax">(...));</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">Problems::issue_problem_segment</span><span class="plain-syntax">(</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"The project says that its syntax is written in a language "</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"other than English (specifically, %1), but the language bundle "</span>
+<span class="plain-syntax">                            </span><span class="string-syntax">"for that language does not provide a file of Preform definitions."</span><span class="plain-syntax">);</span>
+<span class="plain-syntax">                        </span><span class="identifier-syntax">Problems::issue_problem_end</span><span class="plain-syntax">();</span>
+<span class="plain-syntax">                        </span><span class="reserved-syntax">break</span><span class="plain-syntax">;</span>
 <span class="plain-syntax">                    </span><span class="identifier-syntax">default:</span>
 <span class="plain-syntax">                        </span><span class="identifier-syntax">internal_error</span><span class="plain-syntax">(</span><span class="string-syntax">"unknown syntax error"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">                }</span>

docs/inbuild/1-mn.html

@@ -265,10 +265,10 @@ <h1><a href="../index.html">
 </p>
 
 <pre class="displayed-code all-displayed-code code-font">
-<span class="reserved-syntax">void</span><span class="plain-syntax"> </span><span class="function-syntax">Main::load_preform</span><button class="popup" onclick="togglePopup('usagePopup1')"><span class="comment-syntax">?</span><span class="popuptext" id="usagePopup1">Usage of <span class="code-font"><span class="function-syntax">Main::load_preform</span></span>:<br/><a href="1-mn.html#SP2_2">&#167;2.2</a></span></button><span class="plain-syntax">(</span><span class="reserved-syntax">inform_language</span><span class="plain-syntax"> *</span><span class="identifier-syntax">L</span><span class="plain-syntax">) {</span>
+<span class="reserved-syntax">int</span><span class="plain-syntax"> </span><span class="function-syntax">Main::load_preform</span><button class="popup" onclick="togglePopup('usagePopup1')"><span class="comment-syntax">?</span><span class="popuptext" id="usagePopup1">Usage of <span class="code-font"><span class="function-syntax">Main::load_preform</span></span>:<br/><a href="1-mn.html#SP2_2">&#167;2.2</a></span></button><span class="plain-syntax">(</span><span class="reserved-syntax">inform_language</span><span class="plain-syntax"> *</span><span class="identifier-syntax">L</span><span class="plain-syntax">) {</span>
 <span class="plain-syntax">    </span><span class="reserved-syntax">pathname</span><span class="plain-syntax"> *</span><span class="identifier-syntax">P</span><span class="plain-syntax"> = </span><a href="../../../inweb/foundation-module/3-pth.html#SP4" class="function-link"><span class="function-syntax">Pathnames::down</span></a><span class="plain-syntax">(</span><span class="identifier-syntax">path_to_inbuild</span><span class="plain-syntax">, </span><span class="identifier-syntax">I</span><span class="string-syntax">"Tangled"</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">    </span><span class="reserved-syntax">filename</span><span class="plain-syntax"> *</span><span class="identifier-syntax">S</span><span class="plain-syntax"> = </span><a href="../../../inweb/foundation-module/3-fln.html#SP2" class="function-link"><span class="function-syntax">Filenames::in</span></a><span class="plain-syntax">(</span><span class="identifier-syntax">P</span><span class="plain-syntax">, </span><span class="identifier-syntax">I</span><span class="string-syntax">"Syntax.preform"</span><span class="plain-syntax">);</span>
-<span class="plain-syntax">    </span><a href="../words-module/4-lp.html#SP1" class="function-link"><span class="function-syntax">LoadPreform::load</span></a><span class="plain-syntax">(</span><span class="identifier-syntax">S</span><span class="plain-syntax">, </span><span class="identifier-syntax">NULL</span><span class="plain-syntax">);</span>
+<span class="plain-syntax">    </span><span class="reserved-syntax">return</span><span class="plain-syntax"> </span><a href="../words-module/4-lp.html#SP1" class="function-link"><span class="function-syntax">LoadPreform::load</span></a><span class="plain-syntax">(</span><span class="identifier-syntax">S</span><span class="plain-syntax">, </span><span class="identifier-syntax">NULL</span><span class="plain-syntax">);</span>
 <span class="plain-syntax">}</span>
 </pre>
 <p class="commentary firstcommentary"><a id="SP2_6" class="paragraph-anchor"></a><b>&#167;2.6. Target list. </b>This where we keep the list of targets, in which no copy occurs more than