Created unified text file tokenizer

doc/angelscript/Script2Game/GenericDocReaderClass.h

@@ -0,0 +1,55 @@
+namespace Script2Game {
+
+/** \addtogroup ScriptSideAPIs
+ *  @{
+ */    
+
+/** \addtogroup Script2Game
+ *  @{
+ */   
+
+enum TokenType
+{
+    TOKEN_TYPE_NONE,
+    TOKEN_TYPE_LINEBREAK,
+    TOKEN_TYPE_COMMENT,
+    TOKEN_TYPE_STRING,
+    TOKEN_TYPE_NUMBER,
+    TOKEN_TYPE_BOOL,
+    TOKEN_TYPE_KEYWORD
+};
+
+/**
+ * @brief Binding of RoR::GenericDocReader; Traverses document tokens; See 'demo_script.as' for an example.
+ */
+class GenericDocReaderClass
+{
+    GenericDocReader(GenericDocumentPtr@ d);
+
+    bool MoveNext();
+    uint GetPos();
+    bool SeekNextLine();
+    int CountLineArgs();
+    bool EndOfFile(int offset = 0);
+
+    TokenType GetTokType(int offset = 0);
+    string GetStringData(int offset = 0);
+    float GetFloatData(int offset = 0);
+
+    string GetTokString(int offset = 0);
+    float GetTokFloat(int offset = 0);
+    bool GetTokBool(int offset = 0);
+    string GetTokKeyword(int offset = 0);
+    string GetTokComment(int offset = 0);
+
+    bool IsTokString(int offset = 0);
+    bool IsTokFloat(int offset = 0);
+    bool IsTokBool(int offset = 0);
+    bool IsTokKeyword(int offset = 0);
+    bool IsTokComment(int offset = 0);
+};
+
+/// @}    //addtogroup Script2Game
+/// @}    //addtogroup ScriptSideAPIs
+
+} //namespace Script2Game

doc/angelscript/Script2Game/GenericDocumentClass.h

@@ -0,0 +1,42 @@
+namespace Script2Game {
+
+/** \addtogroup ScriptSideAPIs
+ *  @{
+ */    
+
+/** \addtogroup Script2Game
+ *  @{
+ */   
+
+enum GenericDocumentOptions
+{
+    GENERIC_DOCUMENT_OPTION_ALLOW_NAKED_STRINGS, //!< Allow strings without quotes, for backwards compatibility.
+    GENERIC_DOCUMENT_OPTION_ALLOW_SLASH_COMMENTS, //!< Allow comments starting with `//`. 
+    GENERIC_DOCUMENT_OPTION_FIRST_LINE_IS_TITLE, //!< First non-empty & non-comment line is a naked string with spaces. 
+    GENERIC_DOCUMENT_OPTION_ALLOW_SEPARATOR_COLON, //!< Allow ':' as separator between tokens.
+    GENERIC_DOCUMENT_OPTION_PARENTHESES_CAPTURE_SPACES, //!< If non-empty NAKED string encounters '(', following spaces will be captured until matching ')' is found.    
+    GENERIC_DOCUMENT_OPTION_ALLOW_BRACED_KEYWORDS, //!< Allow INI-like '[keyword]' tokens.
+    GENERIC_DOCUMENT_OPTION_ALLOW_SEPARATOR_EQUALS, //!< Allow '=' as separator between tokens.
+    GENERIC_DOCUMENT_OPTION_ALLOW_HASH_COMMENTS //!< Allow comments starting with `#`.     
+};
+
+/**
+ * @brief Binding of RoR::GenericDocument; Parses TRUCK/TOBJ/ODEF/CHARACTER file formats.
+ */
+class GenericDocumentClass
+{
+    /**
+    * Loads and parses a document from OGRE resource system.
+    */
+    bool LoadFromResource(string resource_name, string resource_group_name, int options = 0);
+
+    /**
+    * Saves the document to OGRE resource system.
+    */
+    bool SaveToResource(string resource_name, string resource_group_name);
+};
+
+/// @}    //addtogroup Script2Game
+/// @}    //addtogroup ScriptSideAPIs
+
+} //namespace Script2Game

doc/angelscript/Script2Game/TerrainClass.h

@@ -21,6 +21,16 @@ class TerrainClass
      */
     string getTerrainName();
 
+    /**
+     * @return File name of the terrain definition (TERRN2 format).
+     */
+    string getTerrainFileName(); 
+
+    /**
+     * @return OGRE resource group of the terrain bundle (ZIP/directory under 'mods/') where definition files live.
+     */
+    string getTerrainFileResourceGroup();    
+
     /**
      * @return GUID (global unique ID) of the terrain, or empty string if not specified.
      */

resources/scripts/demo_script.as

@@ -9,6 +9,7 @@
      * Collect and show stats (i.e. frame count, total time)
      * Read/Write cvars (RoR.cfg values, cli args, game state...)
      * View and update game state (current vehicle...)
+     * Parse and display definition files with syntax highlighting.
 
     There are 3 ways of invoking a script:
      1. By defining it with terrain, see terrn2 fileformat, section '[Scripts]':
@@ -21,13 +22,13 @@
 
     For introduction to game events, read
     https://docs.rigsofrods.org/terrain-creation/scripting/.
-
-    For reference manual of the interface, see
-    https://github.com/RigsOfRods/rigs-of-rods/tree/master/doc/angelscript.
+
+    Scripting documentation:
+    https://developer.rigsofrods.org/d4/d07/group___script2_game.html
+
     ---------------------------------------------------------------------------
 */
 
-
 /*
     ---------------------------------------------------------------------------
     Global variables
@@ -38,6 +39,9 @@ CVarClass@  g_app_state = console.cVarFind("app_state"); // 0=bootstrap, 1=main
 CVarClass@  g_sim_state = console.cVarFind("sim_state"); // 0=off, 1=running, 2=paused, 3=terrain editor, see SimState in Application.h
 CVarClass@  g_mp_state = console.cVarFind("mp_state"); // 0=disabled, 1=connecting, 2=connected, see MpState in Application.h
 CVarClass@  g_io_arcade_controls = console.cVarFind("io_arcade_controls"); // bool
+GenericDocumentClass@ g_displayed_document = null;
+string g_displayed_doc_filename;
+array<string> g_terrain_tobj_files;
 
 /*
     ---------------------------------------------------------------------------
@@ -56,8 +60,7 @@ void main()
 void frameStep(float dt)
 {
     // Open demo window
-    ImGui::SetNextWindowSize(vector2(400, 320));
-    ImGui::Begin("Demo Script", /*open:*/true, /*flags:*/0);
+    ImGui::Begin("Demo Script", /*open:*/true, ImGuiWindowFlags_AlwaysAutoResize);
 
     // show some stats
     ImGui::Text("Total frames: " + g_total_frames);
@@ -71,6 +74,11 @@ void frameStep(float dt)
         ImGui::Text("Pro tip: Press '"
             + inputs.getEventCommandTrimmed(EV_COMMON_CONSOLE_TOGGLE)
             + "' to open console anytime.");
+
+        // Reset simulation data
+        @g_displayed_document = null;
+        g_displayed_doc_filename = "";
+        g_terrain_tobj_files.removeRange(0, g_terrain_tobj_files.length());
     }
     else if (g_app_state.getInt() == 2) // simulation
     {
@@ -94,15 +102,48 @@ void frameStep(float dt)
             ImGui::Text("(terrain edit)");
         }
 
+        drawTerrainButtons();
+
         ImGui::TextDisabled("Camera controls:");
         ImGui::Text("Change camera: " + inputs.getEventCommandTrimmed(EV_CAMERA_CHANGE));
         ImGui::Text("Toggle free camera: " + inputs.getEventCommandTrimmed(EV_CAMERA_FREE_MODE));
         ImGui::Text("Toggle fixed camera: " + inputs.getEventCommandTrimmed(EV_CAMERA_FREE_MODE_FIX));              
 
         BeamClass@ actor = game.getCurrentTruck();
-        if (actor != null)
+        if (@actor != null)
         {
+            // Actor name and "View document" button
+            ImGui::PushID("actor");
+            ImGui::AlignTextToFramePadding();
             ImGui::Text("You are driving " + actor.getTruckName());
+            ImGui::SameLine();
+            if (@g_displayed_document == null)
+            {
+                if (ImGui::Button("View document"))
+                {
+                    GenericDocumentClass@ doc = GenericDocumentClass();
+                    int flags = GENERIC_DOCUMENT_OPTION_ALLOW_NAKED_STRINGS
+                              | GENERIC_DOCUMENT_OPTION_ALLOW_SLASH_COMMENTS
+                              | GENERIC_DOCUMENT_OPTION_FIRST_LINE_IS_TITLE
+                              | GENERIC_DOCUMENT_OPTION_ALLOW_SEPARATOR_COLON
+                              | GENERIC_DOCUMENT_OPTION_PARENTHESES_CAPTURE_SPACES;
+                    if (doc.LoadFromResource(actor.getTruckFileName(), actor.getTruckFileResourceGroup(), flags))
+                    {
+                        @g_displayed_document = @doc;
+                        g_displayed_doc_filename = actor.getTruckFileName();
+                    }
+                }
+            }
+            else
+            {
+                if (ImGui::Button("Close document"))
+                {
+                    @g_displayed_document = null;
+                    g_displayed_doc_filename = "";
+                }
+            }
+            ImGui::PopID(); //"actor"
+
             ImGui::TextDisabled("Vehicle controls:");
 
             ImGui::Text("Accelerate/Brake: "
@@ -157,4 +198,159 @@ void frameStep(float dt)
     // Update global counters
     g_total_frames++;
     g_total_seconds += dt;
+
+    // Draw document window
+    if (@g_displayed_document != null)
+    {
+        drawDocumentWindow();
+    }
+}
+
+void drawTerrainButtons()
+{
+    // Terrain name (with "view document" button)
+    ImGui::PushID("terrn");
+    TerrainClass@ terrain = game.getTerrain();
+    ImGui::AlignTextToFramePadding();
+    ImGui::Text("Terrain: " + terrain.getTerrainName());
+    ImGui::SameLine();
+    if (@g_displayed_document == null)
+    {
+        if (ImGui::Button("View document"))
+        {
+            GenericDocumentClass@ doc = GenericDocumentClass();
+            int flags = GENERIC_DOCUMENT_OPTION_ALLOW_NAKED_STRINGS
+                      | GENERIC_DOCUMENT_OPTION_ALLOW_SLASH_COMMENTS
+                      | GENERIC_DOCUMENT_OPTION_ALLOW_HASH_COMMENTS
+                      | GENERIC_DOCUMENT_OPTION_ALLOW_SEPARATOR_EQUALS
+                      | GENERIC_DOCUMENT_OPTION_ALLOW_BRACED_KEYWORDS;
+            if (doc.LoadFromResource(terrain.getTerrainFileName(), terrain.getTerrainFileResourceGroup(), flags))
+            {
+                @g_displayed_document = @doc;
+                g_displayed_doc_filename = terrain.getTerrainFileName();
+
+                // Fetch TOBJ filenames
+                if (g_terrain_tobj_files.length() == 0)
+                {
+                    GenericDocReaderClass@ reader = GenericDocReaderClass(doc);
+                    bool in_section_objects = false;
+                    while (!reader.EndOfFile())
+                    {
+                        if (reader.GetTokType() == TOKEN_TYPE_KEYWORD && reader.GetTokKeyword().substr(0, 1) == "[")
+                        {
+                            in_section_objects = (reader.GetTokKeyword() == '[Objects]');
+                        }
+                        else if (reader.GetTokType() == TOKEN_TYPE_STRING && in_section_objects)
+                        {
+                            // Note: in GenericDocument, a text on line start is always a KEYWORD token,
+                            // but KEYWORDs must not contain special characters,
+                            // so file names always decay to strings because of '.'.                    
+                            g_terrain_tobj_files.insertLast(reader.GetTokString());
+                        }
+                        reader.MoveNext();
+                    }
+                }
+            }
+        }
+    }
+    else
+    {
+        if (ImGui::Button("Close document"))
+        {
+            @g_displayed_document = null;
+            g_displayed_doc_filename = "";
+        }
+    }
+
+    // TOBJ files
+    ImGui::PushID("tobj");
+    for (uint i = 0; i < g_terrain_tobj_files.length(); i++)
+    {
+        ImGui::PushID(i);
+        ImGui::AlignTextToFramePadding();
+        ImGui::Bullet();
+        ImGui::SameLine();
+        ImGui::Text(g_terrain_tobj_files[i]);
+        ImGui::SameLine();
+        if (@g_displayed_document == null)
+        {
+            if (ImGui::Button("View document"))
+            {
+                GenericDocumentClass@ doc = GenericDocumentClass();
+                int flags = GENERIC_DOCUMENT_OPTION_ALLOW_NAKED_STRINGS
+                          | GENERIC_DOCUMENT_OPTION_ALLOW_SLASH_COMMENTS;
+                if (doc.LoadFromResource(g_terrain_tobj_files[i], terrain.getTerrainFileResourceGroup(), flags))
+                {
+                    @g_displayed_document = @doc;   
+                    g_displayed_doc_filename = g_terrain_tobj_files[i];
+                }
+            }
+        }
+        else
+        {
+            if (ImGui::Button("Close document"))
+            {
+                @g_displayed_document = null;
+                g_displayed_doc_filename = "";
+            }
+        }
+        ImGui::PopID(); // i
+    }
+    ImGui::PopID(); //"tobj"
+
+    ImGui::PopID(); //"terrn"
+}
+
+void drawDocumentWindow()
+{
+    ImGui::PushID("document view");
+    string caption = "Document view (" + g_displayed_doc_filename + ")";
+    ImGui::Begin(caption, /*open:*/true, /*flags:*/0);
+
+    GenericDocReaderClass reader(g_displayed_document);
+    while (!reader.EndOfFile())
+    {
+        switch (reader.GetTokType())
+        {
+        // These tokens are always at start of line
+        case TOKEN_TYPE_KEYWORD:
+            ImGui::TextColored(color(1.f, 1.f, 0.f, 1.f), reader.GetTokKeyword());
+            break;
+        case TOKEN_TYPE_COMMENT:
+            ImGui::TextDisabled(";" + reader.GetTokComment());
+            break;
+
+        // Linebreak is implicit in DearIMGUI, no action needed
+        case TOKEN_TYPE_LINEBREAK:
+            break;
+
+        // Other tokens come anywhere - delimiting logic is needed
+        default:
+            if (reader.GetPos() != 0 && reader.GetTokType(-1) != TOKEN_TYPE_LINEBREAK)
+            {
+                ImGui::SameLine();
+                string delimiter = (reader.GetTokType(-1) == TOKEN_TYPE_KEYWORD) ? " " : ", ";
+                ImGui::Text(delimiter);
+                ImGui::SameLine();
+            }
+
+            switch (reader.GetTokType())
+            {
+            case TOKEN_TYPE_STRING:
+                ImGui::TextColored(color(0.f, 1.f, 1.f, 1.f), "\"" + reader.GetTokString() + "\"");
+                break;
+            case TOKEN_TYPE_NUMBER:
+                ImGui::Text("" + reader.GetTokFloat());
+                break;
+            case TOKEN_TYPE_BOOL:
+                ImGui::TextColored(color(1.f, 0.f, 1.f, 1.f), ""+reader.GetTokBool());
+                break;
+            }
+        }
+
+        reader.MoveNext();
+    }
+
+    ImGui::End();
+    ImGui::PopID(); //"document view"
 }