Skip to content

Commit

Permalink
FIX: don't index urls to local files
Browse files Browse the repository at this point in the history
  • Loading branch information
ZogStriP authored and SamSaffron committed Sep 14, 2018
1 parent 74eec18 commit 39a2d92
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 3 deletions.
8 changes: 5 additions & 3 deletions app/services/search_indexer.rb
Expand Up @@ -167,6 +167,8 @@ def self.index(obj, force: false)

class HtmlScrubber < Nokogiri::XML::SAX::Document

DIACRITICS ||= /([\u0300-\u036f]|[\u1AB0-\u1AFF]|[\u1DC0-\u1DFF]|[\u20D0-\u20FF])/

def self.strip_diacritics(str)
s = str.unicode_normalize(:nfkd)
s.gsub!(DIACRITICS, "")
Expand Down Expand Up @@ -196,12 +198,12 @@ def start_element(_, attributes = [])
attributes = Hash[*attributes.flatten]

ATTRIBUTES.each do |name|
characters(attributes[name]) if attributes[name].present?
if attributes[name].present?
characters(attributes[name]) unless name == "href" && UrlHelper.is_local(attributes[name])
end
end
end

DIACRITICS ||= /([\u0300-\u036f]|[\u1AB0-\u1AFF]|[\u1DC0-\u1DFF]|[\u20D0-\u20FF])/

def characters(str)
str = HtmlScrubber.strip_diacritics(str) if @strip_diacritics
scrubbed << " #{str} "
Expand Down
22 changes: 22 additions & 0 deletions spec/services/search_indexer_spec.rb
Expand Up @@ -2,6 +2,7 @@

describe SearchIndexer do
let(:post_id) { 99 }

it 'correctly indexes chinese' do
SiteSetting.default_locale = 'zh_CN'
data = "你好世界"
Expand Down Expand Up @@ -37,6 +38,27 @@
expect(scrubbed).to eq(" HELLO Heterogeneite Здравствуите هتاف للترحيب 你好 ")
end

it "doesn't index local files" do
html = <<~HTML
<p><img src="https://www.discourse.org/logo.png" alt="Discourse"></p>
<p><img src="#{Discourse.base_url_no_prefix}/uploads/episodeinteractive/original/3X/0/f/0f40b818356bdc1d80acfa905034e95cfd112a3a.png" alt="51%20PM" width="289" height="398"></p>
<div class="lightbox-wrapper">
<a class="lightbox" href="#{Discourse.base_url_no_prefix}/uploads/episodeinteractive/original/3X/1/6/16790095df3baf318fb2eb1d7e5d7860dc45d48b.jpg" data-download-href="#{Discourse.base_url_no_prefix}/uploads/episodeinteractive/16790095df3baf318fb2eb1d7e5d7860dc45d48b" title="Untitled design (21).jpg" rel="nofollow noopener">
<img src="#{Discourse.base_url_no_prefix}/uploads/episodeinteractive/optimized/3X/1/6/16790095df3baf318fb2eb1d7e5d7860dc45d48b_1_563x500.jpg" alt="Untitled%20design%20(21)" width="563" height="500">
<div class="meta">
<span class="filename">Untitled design (21).jpg</span>
<span class="informations">1280x1136 472 KB</span>
<span class="expand"></span>
</div>
</a>
</div>
HTML

scrubbed = SearchIndexer::HtmlScrubber.scrub(html).gsub(/\s+/, " ")

expect(scrubbed).to eq(" Discourse 51%20PM Untitled design (21).jpg Untitled%20design%20(21) Untitled design (21).jpg 1280x1136 472 KB ")
end

it 'correctly indexes a post according to version' do
# Preparing so that they can be indexed to right version
SearchIndexer.update_posts_index(post_id, "dummy", "", nil, nil)
Expand Down

0 comments on commit 39a2d92

Please sign in to comment.