Provide options for reducing size

.github/workflows/main.yml

@@ -55,6 +55,7 @@ jobs:
       run: bundle exec rake typecheck:steep
     - name: Check field kinds
       run: rm lib/prism/node.rb && CHECK_FIELD_KIND=true bundle exec rake
+
   build:
     strategy:
       fail-fast: false
@@ -94,11 +95,11 @@ jobs:
       - name: Run Ruby tests
         run: bundle exec rake
 
-  build-debug-mode:
+  build-without-assertions:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, ubuntu-22.04]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
@@ -108,15 +109,13 @@ jobs:
         ruby-version: head
         bundler-cache: true
     - name: Run Ruby tests
-      run: bundle exec rake
-      env:
-        PRISM_DEBUG_MODE_BUILD: "1"
+      run: bundle exec rake compile_no_debug && bundle exec rake test
 
-  build-without-assertions:
+  build-debug:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
@@ -126,7 +125,23 @@ jobs:
         ruby-version: head
         bundler-cache: true
     - name: Run Ruby tests
-      run: bundle exec rake compile_no_debug
+      run: bundle exec rake
+      env:
+        PRISM_BUILD_DEBUG: "1"
+
+  build-minimal:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: head
+        bundler-cache: true
+    - name: Run Ruby tests
+      run: bundle exec rake compile_minimal && bundle exec rake test
+      env:
+        PRISM_BUILD_MINIMAL: "1"
 
   build-java:
     runs-on: ubuntu-latest

Makefile

@@ -94,6 +94,9 @@ all-no-debug: DEBUG_FLAGS := -DNDEBUG=1
 all-no-debug: OPTFLAGS := -O3
 all-no-debug: all
 
+minimal: CFLAGS := $(CFLAGS) -DPRISM_EXCLUDE_SERIALIZATION -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_ENCODING_EXCLUDE_FULL
+minimal: all
+
 run: Makefile $(STATIC_OBJECTS) $(HEADERS) test.c
 	$(ECHO) "compiling test.c"
 	$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) $(STATIC_OBJECTS) test.c

Rakefile

@@ -6,6 +6,7 @@ require "rake/clean"
 
 task compile: :make
 task compile_no_debug: :make_no_debug
+task compile_minimal: :make_minimal
 
 task default: [:compile, :test]
 
@@ -14,13 +15,10 @@ require_relative "templates/template"
 desc "Generate all ERB template based files"
 task templates: Prism::Template::TEMPLATES
 
-task make: [:templates] do
-  sh(RUBY_PLATFORM.include?("openbsd") ? "gmake" : "make")
-end
-
-task make_no_debug: [:templates] do
-  sh("#{RUBY_PLATFORM.include?("openbsd") ? "gmake" : "make"} all-no-debug")
-end
+make = RUBY_PLATFORM.include?("openbsd") ? "gmake" : "make"
+task(make: :templates) { sh(make) }
+task(make_no_debug: :templates) { sh("#{make} all-no-debug") }
+task(make_minimal: :templates) { sh("#{make} minimal") }
 
 # decorate the gem build task with prerequisites
 task build: [:check_manifest, :templates]

docs/build_system.md

@@ -108,3 +108,14 @@ If you want to build prism as a shared library and link against it, you should c
 ```
 MAKEFLAGS="-j10" bundle exec rake compile
 ```
+
+## Build options
+
+* `PRISM_BUILD_DEBUG` - Will cause all file reading to copy into its own allocation to allow easier tracking of reading off the end of the buffer. By default this is off.
+* `PRISM_ENCODING_EXCLUDE_FULL` - Will cause the library to exclude the full encoding API, and only include the minimal number of encodings to support parsing Ruby code without encoding comments. By default this is off.
+* `PRISM_EXPORT_SYMBOLS` - Will cause the shared library to export symbols. By default this is off.
+* `PRISM_EXCLUDE_JSON` - Will cause the library to exclude the JSON API. By default this is off.
+* `PRISM_EXCLUDE_PACK` - Will cause the library to exclude the pack API. By default this is off.
+* `PRISM_EXCLUDE_PRETTYPRINT` - Will cause the library to exclude the prettyprint API. By default this is off.
+* `PRISM_EXCLUDE_SERIALIZATION` - Will cause the library to exclude the serialization API. By default this is off.
+* `PRISM_XALLOCATOR` - Will cause the library to use the custom memory allocator. By default this is off.

ext/prism/extconf.rb

@@ -8,14 +8,14 @@
 
           --enable-debug-mode-build
               Enable debug mode build.
-              You may also use set PRISM_DEBUG_MODE_BUILD environment variable.
+              You may also use set PRISM_BUILD_DEBUG environment variable.
 
           --help
               Display this message.
 
       Environment variables used:
 
-          PRISM_DEBUG_MODE_BUILD
+          PRISM_BUILD_DEBUG
               Equivalent to `--enable-debug-mode-build` when set, even if nil or blank.
 
   TEXT
@@ -72,12 +72,12 @@ def make(target)
 end
 
 # If `--enable-debug-mode-build` is passed to this script or the
-# `PRISM_DEBUG_MODE_BUILD` environment variable is defined, we'll build with the
-# `PRISM_DEBUG_MODE_BUILD` macro defined. This causes parse functions to
+# `PRISM_BUILD_DEBUG` environment variable is defined, we'll build with the
+# `PRISM_BUILD_DEBUG` macro defined. This causes parse functions to
 # duplicate their input so that they have clearly set bounds, which is useful
 # for finding bugs that cause the parser to read off the end of the input.
-if enable_config("debug-mode-build", ENV["PRISM_DEBUG_MODE_BUILD"] || false)
-  append_cflags("-DPRISM_DEBUG_MODE_BUILD")
+if enable_config("debug-mode-build", ENV["PRISM_BUILD_DEBUG"] || false)
+  append_cflags("-DPRISM_BUILD_DEBUG")
 end
 
 # By default, all symbols are hidden in the shared library.

ext/prism/extension.c

@@ -311,7 +311,7 @@ dump(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
     char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
@@ -320,7 +320,7 @@ dump(int argc, VALUE *argv, VALUE self) {
 
     VALUE value = dump_input(&input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     xfree(dup);
 #endif
 
@@ -737,7 +737,7 @@ parse(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
     char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
@@ -746,7 +746,7 @@ parse(int argc, VALUE *argv, VALUE self) {
 
     VALUE value = parse_input(&input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     xfree(dup);
 #endif
 

gemfiles/truffleruby/Gemfile

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-ruby "~> 3.2.2", engine: "truffleruby", engine_version: "~> 23.1.2"
+ruby "~> 3.2.2", engine: "truffleruby", engine_version: "~> 24.0.0"
 
 gemspec path: "../.."
 

gemfiles/truffleruby/Gemfile.lock

@@ -29,7 +29,7 @@ DEPENDENCIES
   test-unit
 
 RUBY VERSION
-   ruby 3.2.2p0 (truffleruby 23.1.2)
+   ruby 3.2.2p0 (truffleruby 24.0.0)
 
 BUNDLED WITH
    2.2.32

include/prism.h

@@ -98,6 +98,11 @@ typedef char * (pm_parse_stream_fgets_t)(char *string, int size, void *stream);
  */
 PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t *parser, pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *fgets, const pm_options_t *options);
 
+// We optionally support serializing to a binary string. For systems that don't
+// want or need this functionality, it can be turned off with the
+// PRISM_EXCLUDE_SERIALIZATION define.
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
 /**
  * Parse and serialize the AST represented by the source that is read out of the
  * given stream into to the given buffer.
@@ -185,6 +190,8 @@ PRISM_EXPORTED_FUNCTION void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t
  */
 PRISM_EXPORTED_FUNCTION void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
 
+#endif
+
 /**
  * Parse the source and return true if it parses without errors or warnings.
  *
@@ -220,6 +227,10 @@ const char * pm_token_type_human(pm_token_type_t token_type);
  */
 PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
 
+// We optionally support dumping to JSON. For systems that don't want or need
+// this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define.
+#ifndef PRISM_EXCLUDE_JSON
+
 /**
  * Dump JSON to the given buffer.
  *
@@ -229,6 +240,8 @@ PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser,
  */
 PRISM_EXPORTED_FUNCTION void pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
 
+#endif
+
 /**
  * @mainpage
  *

include/prism/encoding.h

@@ -135,7 +135,14 @@ extern const uint8_t pm_encoding_unicode_table[256];
  */
 typedef enum {
     PM_ENCODING_UTF_8 = 0,
+    PM_ENCODING_US_ASCII,
     PM_ENCODING_ASCII_8BIT,
+    PM_ENCODING_EUC_JP,
+    PM_ENCODING_WINDOWS_31J,
+
+// We optionally support excluding the full set of encodings to only support the
+// minimum necessary to process Ruby code without encoding comments.
+#ifndef PRISM_ENCODING_EXCLUDE_FULL
     PM_ENCODING_BIG5,
     PM_ENCODING_BIG5_HKSCS,
     PM_ENCODING_BIG5_UAO,
@@ -148,7 +155,6 @@ typedef enum {
     PM_ENCODING_CP950,
     PM_ENCODING_CP951,
     PM_ENCODING_EMACS_MULE,
-    PM_ENCODING_EUC_JP,
     PM_ENCODING_EUC_JP_MS,
     PM_ENCODING_EUC_JIS_2004,
     PM_ENCODING_EUC_KR,
@@ -208,7 +214,6 @@ typedef enum {
     PM_ENCODING_STATELESS_ISO_2022_JP,
     PM_ENCODING_STATELESS_ISO_2022_JP_KDDI,
     PM_ENCODING_TIS_620,
-    PM_ENCODING_US_ASCII,
     PM_ENCODING_UTF8_MAC,
     PM_ENCODING_UTF8_DOCOMO,
     PM_ENCODING_UTF8_KDDI,
@@ -222,8 +227,9 @@ typedef enum {
     PM_ENCODING_WINDOWS_1256,
     PM_ENCODING_WINDOWS_1257,
     PM_ENCODING_WINDOWS_1258,
-    PM_ENCODING_WINDOWS_31J,
     PM_ENCODING_WINDOWS_874,
+#endif
+
     PM_ENCODING_MAXIMUM
 } pm_encoding_type_t;
 

include/prism/pack.h

@@ -6,6 +6,15 @@
 #ifndef PRISM_PACK_H
 #define PRISM_PACK_H
 
+// We optionally support parsing String#pack templates. For systems that don't
+// want or need this functionality, it can be turned off with the
+// PRISM_EXCLUDE_PACK define.
+#ifdef PRISM_EXCLUDE_PACK
+
+void pm_pack_parse(void);
+
+#else
+
 #include "prism/defines.h"
 
 #include <stdint.h>
@@ -150,3 +159,5 @@ pm_pack_parse(
 PRISM_EXPORTED_FUNCTION size_t pm_size_to_native(pm_pack_size size);
 
 #endif
+
+#endif

include/prism/prettyprint.h

@@ -6,6 +6,12 @@
 #ifndef PRISM_PRETTYPRINT_H
 #define PRISM_PRETTYPRINT_H
 
+#ifdef PRISM_EXCLUDE_PRETTYPRINT
+
+void pm_prettyprint(void);
+
+#else
+
 #include "prism/defines.h"
 
 #include <stdio.h>
@@ -24,3 +30,5 @@
 PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
 
 #endif
+
+#endif

rakelib/test.rake

@@ -33,8 +33,8 @@ namespace :test do
 
     desc "Run tests under valgrind"
     task :valgrind do
-      # Recompile with PRISM_DEBUG_MODE_BUILD=1
-      ENV["PRISM_DEBUG_MODE_BUILD"] = "1"
+      # Recompile with PRISM_BUILD_DEBUG=1
+      ENV["PRISM_BUILD_DEBUG"] = "1"
       Rake::Task["clobber"].invoke
       Rake::Task["test:valgrind_internal"].invoke
     end