FEATURE: Adds support for local js files in themes

app/models/javascript_cache.rb

@@ -8,11 +8,19 @@ class JavascriptCache < ActiveRecord::Base
   before_save :update_digest
 
   def url
-    "#{GlobalSetting.cdn_url}#{Discourse.base_path}/theme-javascripts/#{digest}.js?__ws=#{Discourse.current_hostname}"
+    "#{GlobalSetting.cdn_url}#{Discourse.base_path}#{path}"
+  end
+
+  def local_url
+    "#{Discourse.base_url}#{path}"
   end
 
   private
 
+  def path
+    "/theme-javascripts/#{digest}.js?__ws=#{Discourse.current_hostname}"
+  end
+
   def update_digest
     self.digest = Digest::SHA1.hexdigest(content) if content_changed?
   end

app/models/theme.rb

@@ -28,6 +28,7 @@ class Theme < ActiveRecord::Base
   has_one :javascript_cache, dependent: :destroy
   has_many :locale_fields, -> { filter_locale_fields(I18n.fallbacks[I18n.locale]) }, class_name: 'ThemeField'
   has_many :upload_fields, -> { where(type_id: ThemeField.types[:theme_upload_var]).preload(:upload) }, class_name: 'ThemeField'
+  has_many :raw_js_fields, -> { where(target_id: Theme.targets[:raw_js]).preload(:javascript_cache) }, class_name: 'ThemeField'
   has_many :extra_scss_fields, -> { where(target_id: Theme.targets[:extra_scss]) }, class_name: 'ThemeField'
   has_many :yaml_theme_fields, -> { where("name = 'yaml' AND type_id = ?", ThemeField.types[:yaml]) }, class_name: 'ThemeField'
   has_many :var_theme_fields, -> { where("type_id IN (?)", ThemeField.theme_var_type_ids) }, class_name: 'ThemeField'
@@ -339,7 +340,17 @@ def self.clear_cache!
   end
 
   def self.targets
-    @targets ||= Enum.new(common: 0, desktop: 1, mobile: 2, settings: 3, translations: 4, extra_scss: 5, extra_js: 6, tests_js: 7)
+    @targets ||= Enum.new(
+      common: 0,
+      desktop: 1,
+      mobile: 2,
+      settings: 3,
+      translations: 4,
+      extra_scss: 5,
+      extra_js: 6,
+      tests_js: 7,
+      raw_js: 8
+    )
   end
 
   def self.lookup_target(target_id)
@@ -546,6 +557,17 @@ def build_settings_hash
     end
     hash['theme_uploads'] = theme_uploads if theme_uploads.present?
 
+    raw_js_urls = {}
+    theme_fields.filter(&:raw_js_field?).each do |field|
+      if js_cache = field.javascript_cache
+        raw_js_urls[field.name] = {
+          url: js_cache.url,
+          local_url: js_cache.local_url
+        }
+      end
+    end
+    hash['raw_js'] = raw_js_urls if raw_js_urls.present?
+
     hash
   end
 
@@ -653,6 +675,7 @@ def scss_variables
 
     settings_hash&.each do |name, value|
       next if name == "theme_uploads"
+      next if name == "raw_js"
       contents << to_scss_variable(name, value)
     end
 

app/models/theme_field.rb

@@ -37,13 +37,16 @@ class ThemeField < ActiveRecord::Base
   }
 
   def self.types
-    @types ||= Enum.new(html: 0,
-                        scss: 1,
-                        theme_upload_var: 2,
-                        theme_color_var: 3, # No longer used
-                        theme_var: 4, # No longer used
-                        yaml: 5,
-                        js: 6)
+    @types ||= Enum.new(
+      html: 0,
+      scss: 1,
+      theme_upload_var: 2,
+      theme_color_var: 3, # No longer used
+      theme_var: 4, # No longer used
+      yaml: 5,
+      js: 6,
+      raw_js: 7
+    )
   end
 
   def self.theme_var_type_ids
@@ -184,7 +187,7 @@ def validate_svg_sprite_xml
 
     begin
       content = File.read(path)
-      svg_file = Nokogiri::XML(content) do |config|
+      Nokogiri::XML(content) do |config|
         config.options = Nokogiri::XML::ParseOptions::NOBLANKS
       end
     rescue => e
@@ -311,6 +314,8 @@ def self.guess_type(name:, target:)
       types[:js]
     elsif target.to_s == "settings" || target.to_s == "translations"
       types[:yaml]
+    elsif target.to_s == "raw_js"
+      types[:raw_js]
     end
   end
 
@@ -331,6 +336,10 @@ def basic_html_field?
       ThemeField.html_fields.include?(self.name)
   end
 
+  def raw_js_field?
+    Theme.targets[self.target_id] == :raw_js
+  end
+
   def extra_js_field?
     Theme.targets[self.target_id] == :extra_js
   end
@@ -387,6 +396,12 @@ def ensure_baked!
       self.error = validate_svg_sprite_xml
       self.value_baked = "baked"
       self.compiler_version = Theme.compiler_version
+    elsif raw_js_field?
+      javascript_cache || build_javascript_cache
+      javascript_cache.content = self.value
+      javascript_cache.save!
+      self.value_baked = "baked"
+      self.compiler_version = Theme.compiler_version
     end
 
     if self.will_save_change_to_value_baked? ||
@@ -512,6 +527,9 @@ def filename_from_opts(opts)
     ThemeFileMatcher.new(regex: /^javascripts\/(?<name>.+)$/,
                          targets: :extra_js, names: nil, types: :js,
                          canonical: -> (h) { "javascripts/#{h[:name]}" }),
+    ThemeFileMatcher.new(regex: /^raw-javascripts\/(?<name>.+)$/,
+                         targets: :raw_js, names: nil, types: :js,
+                         canonical: -> (h) { "raw-javascripts/#{h[:name]}" }),
     ThemeFileMatcher.new(regex: /^test\/(?<name>.+)$/,
                          targets: :tests_js, names: nil, types: :js,
                          canonical: -> (h) { "test/#{h[:name]}" }),
@@ -555,6 +573,9 @@ def dependent_fields
     elsif settings_field?
       return theme.theme_fields.where(target_id: ThemeField.basic_targets.map { |t| Theme.targets[t.to_sym] },
                                       name: ThemeField.scss_fields + ThemeField.html_fields)
+    elsif raw_js_field?
+      return theme.theme_fields.where(target_id: ThemeField.basic_targets.map { |t| Theme.targets[t.to_sym] },
+                                      name: ThemeField.html_fields)
     end
     ThemeField.none
   end

config/site_settings.yml

@@ -1274,7 +1274,7 @@ files:
     default: 50000
     max: 1024000
   theme_authorized_extensions:
-    default: "jpg|jpeg|png|woff|woff2|svg|eot|ttf|otf|gif|webp|js"
+    default: "jpg|jpeg|png|woff|woff2|svg|eot|ttf|otf|gif|webp|js|wasm"
     type: list
     list_type: compact
   authorized_extensions:

spec/models/theme_field_spec.rb

@@ -174,7 +174,7 @@
 
   it "correctly handles extra JS fields" do
     js_field = theme.set_field(target: :extra_js, name: "discourse/controllers/discovery.js.es6", value: "import 'discourse/lib/ajax'; console.log('hello from .js.es6');")
-    js_2_field = theme.set_field(target: :extra_js, name: "discourse/controllers/discovery-2.js", value: "import 'discourse/lib/ajax'; console.log('hello from .js');")
+    theme.set_field(target: :extra_js, name: "discourse/controllers/discovery-2.js", value: "import 'discourse/lib/ajax'; console.log('hello from .js');")
     hbs_field = theme.set_field(target: :extra_js, name: "discourse/templates/discovery.hbs", value: "{{hello-world}}")
     raw_hbs_field = theme.set_field(target: :extra_js, name: "discourse/templates/discovery.hbr", value: "{{hello-world}}")
     hbr_field = theme.set_field(target: :extra_js, name: "discourse/templates/other_discovery.hbr", value: "{{hello-world}}")
@@ -200,6 +200,60 @@
     expect(theme.javascript_cache.content).to include("var settings =")
   end
 
+  it "correctly handles raw JS assets" do
+    content = "// not transpiled; console.log('hello world');"
+
+    theme.set_field(target: :extra_js, name: "discourse/controllers/discovery.js", value: "let a = 'b';")
+    common_field = theme.set_field(target: :common, name: "head_tag", value: "<script>let c = 'd';</script>", type: :html)
+    theme.save!
+
+    theme.set_field(target: :settings, type: :yaml, name: "yaml", value: "hello: world")
+    theme.save!
+
+    raw_js_field = theme.set_field(target: :raw_js, name: "test", value: content)
+    theme.save!
+
+    raw_js_field.reload
+    expect(raw_js_field.javascript_cache.content).to eq(content)
+
+    common_field.ensure_baked!
+    common_field.reload
+
+    # a bit fragile, but at least we test it properly
+    [theme.reload.javascript_cache.content, common_field.reload.javascript_cache.content].each do |js|
+      js_to_eval = <<~JS
+        var settings;
+        var window = {};
+        var require = function(name) {
+          if(name == "discourse/lib/theme-settings-store") {
+            return({
+              registerSettings: function(id, s) {
+                settings = s;
+              }
+            });
+          }
+        }
+        window.require = require;
+        #{js}
+        settings
+      JS
+
+      ctx = MiniRacer::Context.new
+      val = ctx.eval(js_to_eval)
+      ctx.dispose
+
+      expect(val["raw_js"]).to eq({
+        "test" => {
+          "local_url" => raw_js_field.javascript_cache.local_url,
+          "url" => raw_js_field.javascript_cache.url
+        }
+      })
+    end
+
+    # this is important, we do not want local_js_urls to leak into scss
+    expect(theme.scss_variables).to eq("$hello: unquote(\"world\");")
+  end
+
   def create_upload_theme_field!(name)
     ThemeField.create!(
       theme_id: 1,

spec/requests/admin/themes_controller_spec.rb

@@ -161,9 +161,14 @@
       json = response.parsed_body
 
       expect(json["theme"]["name"]).to eq("Header Icons")
-      expect(json["theme"]["theme_fields"].length).to eq(5)
+      expect(json["theme"]["theme_fields"].length).to eq(6)
       expect(json["theme"]["auto_update"]).to eq(false)
       expect(UserHistory.where(action: UserHistory.actions[:change_theme]).count).to eq(1)
+
+      theme = Theme.find(json["theme"]["id"])
+
+      # the theme imported includes `raw-javascripts` with a file including this contents
+      expect(theme.raw_js_fields.first.javascript_cache.content).to include("web worker")
     end
 
     it 'updates an existing theme from an archive by name' do
@@ -177,7 +182,7 @@
       json = response.parsed_body
 
       expect(json["theme"]["name"]).to eq("Header Icons")
-      expect(json["theme"]["theme_fields"].length).to eq(5)
+      expect(json["theme"]["theme_fields"].length).to eq(6)
       expect(UserHistory.where(action: UserHistory.actions[:change_theme]).count).to eq(1)
     end
 
@@ -202,7 +207,7 @@
 
       expect(json["theme"]["name"]).to eq("Some other name")
       expect(json["theme"]["id"]).to eq(other_existing_theme.id)
-      expect(json["theme"]["theme_fields"].length).to eq(5)
+      expect(json["theme"]["theme_fields"].length).to eq(6)
       expect(UserHistory.where(action: UserHistory.actions[:change_theme]).count).to eq(1)
     end
 
@@ -218,7 +223,7 @@
 
       expect(json["theme"]["name"]).to eq("Header Icons")
       expect(json["theme"]["id"]).not_to eq(existing_theme.id)
-      expect(json["theme"]["theme_fields"].length).to eq(5)
+      expect(json["theme"]["theme_fields"].length).to eq(6)
       expect(json["theme"]["auto_update"]).to eq(false)
       expect(UserHistory.where(action: UserHistory.actions[:change_theme]).count).to eq(1)
     end