Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge #535
- Loading branch information
Showing
7 changed files
with
106 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| #!/usr/bin/env ruby | ||
|
|
||
| ENV["RAILS_ENV"] ||= "production" | ||
|
|
||
| APP_PATH = File.expand_path('../../config/application', __FILE__) | ||
| require File.expand_path('../../config/boot', __FILE__) | ||
| require APP_PATH | ||
| Rails.application.require_environment! | ||
|
|
||
| LAST_STORY_KEY = "webmentions:last_story_id".freeze | ||
|
|
||
| def endpoint_from_body(html) | ||
| doc = Nokogiri::HTML(html) | ||
|
|
||
| if !doc.css('[rel~="webmention"]').css('[href]').empty? | ||
| doc.css('[rel~="webmention"]').css('[href]').attribute('href').value | ||
| elsif !doc.css('[rel="http://webmention.org/"]').css('[href]').empty? | ||
| doc.css('[rel="http://webmention.org/"]').css('[href]').attribute('href').value | ||
| elsif !doc.css('[rel="http://webmention.org"]').css('[href]').empty? | ||
| doc.css('[rel="http://webmention.org"]').css('[href]').attribute('href').value | ||
| end | ||
| end | ||
|
|
||
| def endpoint_from_headers(header) | ||
| return unless header | ||
|
|
||
| if (matches = header.match(/<([^>]+)>; rel="[^"]*\s?webmention\s?[^"]*"/)) | ||
| return matches[1] | ||
| elsif (matches = header.match(/<([^>]+)>; rel=webmention/)) | ||
| return matches[1] | ||
| elsif (matches = header.match(/rel="[^"]*\s?webmention\s?[^"]*"; <([^>]+)>/)) | ||
| return matches[1] | ||
| elsif (matches = header.match(/rel=webmention; <([^>]+)>/)) | ||
| return matches[1] | ||
| elsif (matches = header.match(/<([^>]+)>; rel="http:\/\/webmention\.org\/?"/)) | ||
| return matches[1] | ||
| elsif (matches = header.match(/rel="http:\/\/webmention\.org\/?"; <([^>]+)>/)) | ||
| return matches[1] | ||
| end | ||
| end | ||
|
|
||
| # Some pages could return a relative link as their webmention endpoint. | ||
| # We need to translate this relative likn to an absolute one. | ||
| def uri_to_absolute(uri, req_uri) | ||
| abs_uri = URI.parse(uri) | ||
| if abs_uri.host | ||
| # Already absolute. | ||
| return uri | ||
| else | ||
| abs_uri.host = req_uri.host | ||
| abs_uri.scheme = req_uri.scheme | ||
| abs_uri.port = req_uri.port | ||
| return abs_uri | ||
| end | ||
| end | ||
|
|
||
| def send_webmention(source, target, endpoint) | ||
| sp = Sponge.new | ||
| sp.timeout = 10 | ||
| sp.fetch(endpoint.to_s, :post, { | ||
| "source" => URI.encode_www_form_component(source), | ||
| "target" => URI.encode_www_form_component(target), | ||
| }, nil, {}, 3) | ||
| end | ||
|
|
||
| if __FILE__ == $PROGRAM_NAME | ||
| last_story_id = (Keystore.value_for(LAST_STORY_KEY) || Story.order('id desc').limit(1).offset(1).pluck(:id).try(:first)).to_i | ||
|
|
||
| Story.where("id > ? AND is_expired = ?", last_story_id, false).order(:id).each do |s| | ||
| next unless s.url | ||
|
|
||
| # mark it done so we don't hit them again if we or they crash | ||
| Keystore.put(LAST_STORY_KEY, s.id) | ||
|
|
||
| sp = Sponge.new | ||
| sp.timeout = 10 | ||
| response = sp.fetch(s.url, :get, nil, nil, { | ||
| "User-agent" => "#{Rails.application.domain} webmention endpoint lookup", | ||
| }, 3) | ||
| next unless response | ||
|
|
||
| wm_endpoint_raw = endpoint_from_headers(response['link']) || | ||
| endpoint_from_body(response.body.to_s) | ||
| next unless wm_endpoint_raw | ||
|
|
||
| wm_endpoint = uri_to_absolute(wm_endpoint_raw, URI.parse(s.url)) | ||
| send_webmention(s.short_id_url, s.url, wm_endpoint) | ||
| last_story_id = s.id | ||
| end | ||
| end |