diff --git a/app/jobs/regular/process_post.rb b/app/jobs/regular/process_post.rb index d6b3afa4d275e1..cd99dec5d63774 100644 --- a/app/jobs/regular/process_post.rb +++ b/app/jobs/regular/process_post.rb @@ -32,7 +32,7 @@ def execute(args) # TODO suicide if needed, let's gather a few here first Rails.logger.warn("Cooked post processor in FATAL state, bypassing. You need to urgently restart sidekiq\norig: #{orig_cooked}\nrecooked: #{recooked}\ncooked: #{cooked}\npost id: #{post.id}") else - post.update_column(:cooked, cp.html) + post.update!(cooked: cp.html) extract_links(post) post.publish_change_to_clients! :revised end diff --git a/app/models/post.rb b/app/models/post.rb index c7382046567b4c..b835fea5a391c4 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -603,7 +603,11 @@ def rebake!(invalidate_broken_images: false, invalidate_oneboxes: false, priorit new_cooked = cook(raw, topic_id: topic_id, invalidate_oneboxes: invalidate_oneboxes) old_cooked = cooked - update_columns(cooked: new_cooked, baked_at: Time.new, baked_version: BAKED_VERSION) + self.update!( + cooked: new_cooked, + baked_at: Time.zone.now, + baked_version: BAKED_VERSION + ) if invalidate_broken_images custom_fields.delete(BROKEN_IMAGES) diff --git a/app/services/search_indexer.rb b/app/services/search_indexer.rb index c7b07c504d5ecb..f731e52e87ec69 100644 --- a/app/services/search_indexer.rb +++ b/app/services/search_indexer.rb @@ -19,11 +19,16 @@ def self.inject_extra_terms(raw) # insert some extra words for I.am.a.word so "word" is tokenized # I.am.a.word becomes I.am.a.word am a word raw.gsub(/[^[:space:]]*[\.]+[^[:space:]]*/) do |with_dot| - split = with_dot.split(".") - if split.length > 1 - with_dot + ((+" ") << split[1..-1].join(" ")) + if with_dot.match?(PlainTextToMarkdown::URL_REGEX) + "#{with_dot} #{URI.parse(with_dot).hostname.gsub('.', ' ')}" else - with_dot + split = with_dot.split(".") + + if split.length > 1 + with_dot + ((+" ") << split[1..-1].join(" ")) + else + with_dot + end end end end @@ -183,19 +188,34 @@ def initialize(strip_diacritics: false) def self.scrub(html, strip_diacritics: false) return +"" if html.blank? + document = Nokogiri::HTML("
#{html}
", nil, Encoding::UTF_8.to_s) + + document.css( + "div.#{CookedPostProcessor::LIGHTBOX_WRAPPER_CSS_CLASS}" + ).remove + + document.css("a[href]").each do |node| + node.remove_attribute("href") if node["href"] == node.text + end + me = new(strip_diacritics: strip_diacritics) - Nokogiri::HTML::SAX::Parser.new(me).parse("
#{html}
") + Nokogiri::HTML::SAX::Parser.new(me).parse(document.to_html) me.scrubbed.squish end ATTRIBUTES ||= %w{alt title href data-youtube-title} - def start_element(_, attributes = []) + def start_element(_name, attributes = []) attributes = Hash[*attributes.flatten] - ATTRIBUTES.each do |name| - if attributes[name].present? - characters(attributes[name]) unless name == "href" && UrlHelper.is_local(attributes[name]) + ATTRIBUTES.each do |attribute_name| + if attributes[attribute_name].present? && + !( + attribute_name == "href" && + UrlHelper.is_local(attributes[attribute_name]) + ) + + characters(attributes[attribute_name]) end end end diff --git a/lib/cooked_post_processor.rb b/lib/cooked_post_processor.rb index 364f1fcd160970..68a3055011ffc3 100644 --- a/lib/cooked_post_processor.rb +++ b/lib/cooked_post_processor.rb @@ -8,6 +8,7 @@ class CookedPostProcessor INLINE_ONEBOX_LOADING_CSS_CLASS = "inline-onebox-loading" INLINE_ONEBOX_CSS_CLASS = "inline-onebox" + LIGHTBOX_WRAPPER_CSS_CLASS = "lightbox-wrapper" LOADING_SIZE = 10 LOADING_COLORS = 32 @@ -367,7 +368,7 @@ def each_responsive_ratio def add_lightbox!(img, original_width, original_height, upload, cropped: false) # first, create a div to hold our lightbox - lightbox = create_node("div", "lightbox-wrapper") + lightbox = create_node("div", LIGHTBOX_WRAPPER_CSS_CLASS) img.add_next_sibling(lightbox) lightbox.add_child(img) diff --git a/spec/services/search_indexer_spec.rb b/spec/services/search_indexer_spec.rb index d748ec638a9a93..303458610b116a 100644 --- a/spec/services/search_indexer_spec.rb +++ b/spec/services/search_indexer_spec.rb @@ -61,7 +61,7 @@ def scrub(html, strip_diacritics: false) scrubbed = scrub(html) - expect(scrubbed).to eq("Discourse 51%20PM Untitled design (21).jpg Untitled%20design%20(21) Untitled design (21).jpg 1280x1136 472 KB") + expect(scrubbed).to eq("Discourse 51%20PM") end it 'correctly indexes a post according to version' do @@ -110,5 +110,41 @@ def scrub(html, strip_diacritics: false) post.save!(validate: false) end.to_not change { PostSearchData.count } end + + it "should not tokenize urls and duplicate title and href in " do + post = Fabricate(:post, raw: <<~RAW) + https://meta.discourse.org/some.png + RAW + + post.rebake! + post.reload + topic = post.topic + + expect(post.post_search_data.raw_data).to eq( + "#{topic.title} #{topic.category.name} https://meta.discourse.org/some.png meta discourse org" + ) + end + + it 'should not include lightbox in search' do + Jobs.run_immediately! + SiteSetting.max_image_height = 2000 + SiteSetting.crawl_images = true + FastImage.expects(:size).returns([1750, 2000]) + + src = "https://meta.discourse.org/some.png" + + post = Fabricate(:post, raw: <<~RAW) + Let me see how I can fix this image + + RAW + + post.rebake! + post.reload + topic = post.topic + + expect(post.post_search_data.raw_data).to eq( + "#{topic.title} #{topic.category.name} Let me see how I can fix this image" + ) + end end end