Permalink
Browse files

Support Hibiki renewal

  • Loading branch information...
yayugu committed Nov 9, 2015
1 parent 9050217 commit 8bb01a577c1e9ad39da852502b620ea94d4f0b3a
View
@@ -26,20 +26,25 @@ Net Radio Archive
- LinuxなどUNIX的なOS (Windowsでも動かしたい...)
- Ruby 2.0 or higher
- rtmpdump
-- ffmpeg or livav
- swftools
+- あたらしめのffmpeg (HTTP Live Streaming の input に対応しているもの)
+ - ※最新のffmpegの導入は面倒であることが多いです。自分はLinuxではstatic buildを使っています。 http://qiita.com/yayugu/items/d7f6a15a6f988064f51c
+ - Macではhomebrewで導入できるバージョンで問題ありません
- (AG-ONのみ)
- GUI環境 or xvfb
- firefox
- - とてもあたらしいffmpeg (HTTP Live Streaming の input に対応しているもの)
- - ※最新のffmpegの導入は面倒であることが多いです。自分はLinux用のstatic buildを使っています。 http://qiita.com/yayugu/items/d7f6a15a6f988064f51c
## セットアップ
```
# 必要なライブラリをインストール
# Ubuntuの場合:
-$ sudo apt-get install rtmpdump libav-tools swftools ruby
+$ sudo apt-get install rtmpdump swftools ruby
+
+$ # libavがインストールされている場合には削除してから
+$ wget http://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz
+$ tar xvf ffmpeg-release-64bit-static.tar.xz
+$ sudo cp ./ffmpeg-release-64bit-static/ffmpeg /usr/local/bin
$ git clone https://github.com/yayugu/net-radio-archive.git
$ cd net-radio-archive
@@ -4,6 +4,8 @@ module OndemandRetry
downloading: 'downloading',
done: 'done',
failed: 'failed',
+ not_downloadable: 'not_downloadable',
+ outdated: 'outdated',
}
RETRY_LIMIT = 3
end
@@ -0,0 +1,3 @@
+class HibikiProgramV2 < ActiveRecord::Base
+ include OndemandRetry
+end
@@ -0,0 +1,25 @@
+class CreateHibikiProgramV2 < ActiveRecord::Migration
+ def up
+ sql = <<EOF
+CREATE TABLE `hibiki_program_v2s` (
+ `id` int UNSIGNED NOT NULL AUTO_INCREMENT,
+ `access_id` varchar(100) CHARACTER SET ASCII NOT NULL,
+ `episode_id` int UNSIGNED NOT NULL,
+ `title` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
+ `episode_name` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
+ `cast` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
+ `state` varchar(100) CHARACTER SET ASCII NOT NULL,
+ `retry_count` int UNSIGNED NOT NULL,
+ `created_at` datetime NOT NULL,
+ `updated_at` datetime NOT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY (`access_id`, `episode_id`)
+);
+EOF
+ ActiveRecord::Base.connection.execute(sql)
+ end
+ def down
+ sql = 'DROP TABLE `hibiki_program_v2s`;'
+ ActiveRecord::Base.connection.execute(sql)
+ end
+end
View
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20151030150712) do
+ActiveRecord::Schema.define(version: 20151110023306) do
create_table "agon_programs", force: true do |t|
t.string "title", limit: 250, null: false
@@ -39,6 +39,34 @@
add_index "anitama_programs", ["book_id", "update_time"], name: "book_id", unique: true, using: :btree
+ create_table "hibiki_program_v2", force: true do |t|
+ t.string "access_id", limit: 100, null: false
+ t.integer "episode_id", null: false
+ t.string "title", limit: 250, null: false
+ t.string "episode_name", limit: 250, null: false
+ t.string "cast", limit: 250, null: false
+ t.string "state", limit: 100, null: false
+ t.integer "retry_count", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ end
+
+ add_index "hibiki_program_v2", ["access_id", "episode_id"], name: "access_id", unique: true, using: :btree
+
+ create_table "hibiki_program_v2s", force: true do |t|
+ t.string "access_id", limit: 100, null: false
+ t.integer "episode_id", null: false
+ t.string "title", limit: 250, null: false
+ t.string "episode_name", limit: 250, null: false
+ t.string "cast", limit: 250, null: false
+ t.string "state", limit: 100, null: false
+ t.integer "retry_count", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ end
+
+ add_index "hibiki_program_v2s", ["access_id", "episode_id"], name: "access_id", unique: true, using: :btree
+
create_table "hibiki_programs", force: true do |t|
t.string "title", limit: 250, null: false
t.string "comment", limit: 150, null: false
View
@@ -5,37 +5,71 @@ module Hibiki
class Downloading
CH_NAME = 'hibiki'
+ def initialize
+ @a = Mechanize.new
+ @a.user_agent_alias = 'Windows Chrome'
+ end
+
def download(program)
- unless exec_rec(program)
- return false
+ infos = get_infos(program)
+ if infos['episode']['id'] != program.episode_id
+ Rails.logger.error("episode outdated. title=#{program.title} expected_episode_id=#{program.episode_id} actual_episode_id=#{infos['episode']['id']}")
+ program.state = HibikiProgramV2::STATE[:outdated]
+ return
+ end
+ live_flg = infos['episode'].try(:[], 'video').try(:[], 'live_flg')
+ if live_flg == nil || live_flg == true
+ program.state = HibikiProgramV2::STATE[:not_downloadable]
+ return
end
- exec_convert(program)
+ url = get_m3u8_url(infos['episode']['video']['id'])
+ unless download_hls(program, url)
+ program.state = HibikiProgramV2::STATE[:failed]
+ return
+ end
+ program.state = HibikiProgramV2::STATE[:done]
end
- def exec_rec(program)
- flv_path = Main::file_path_working(CH_NAME, title(program), 'flv')
- command = "rtmpdump -q -r #{Shellwords.escape(program.rtmp_url)} -o #{Shellwords.escape(flv_path)}"
+ def get_infos(program)
+ res = @a.get("https://vcms-api.hibiki-radio.jp/api/v1/programs/#{program.access_id}")
+ infos = JSON.parse(res.body)
+ end
+
+ def get_m3u8_url(video_id)
+ res = @a.get("https://vcms-api.hibiki-radio.jp/api/v1/videos/play_check?video_id=#{video_id}")
+ play_infos = JSON.parse(res.body)
+ play_infos['playlist_url']
+ end
+
+ def download_hls(program, m3u8_url)
+ file_path = Main::file_path_working(CH_NAME, title(program), 'mp4')
+ arg = "\
+ -loglevel error \
+ -y \
+ -i #{Shellwords.escape(m3u8_url)} \
+ -vcodec copy -acodec copy -bsf:a aac_adtstoasc \
+ #{Shellwords.escape(file_path)}"
Main::prepare_working_dir(CH_NAME)
- exit_status, output = Main::shell_exec(command)
+ exit_status, output = Main::ffmpeg(arg)
unless exit_status.success?
- #Rails.logger.error "rec failed. program:#{program}, exit_status:#{exit_status}, output:#{output}"
+ Rails.logger.error "rec failed. program:#{program}, exit_status:#{exit_status}, output:#{output}"
return false
end
- true
- end
+ Main::move_to_archive_dir(CH_NAME, program.created_at, file_path)
- def exec_convert(program)
- flv_path = Main::file_path_working(CH_NAME, title(program), 'flv')
- mp4_path = Main::file_path_working(CH_NAME, title(program), 'mp4')
- Main::convert_ffmpeg_to_mp4(flv_path, mp4_path, program)
- Main::move_to_archive_dir(CH_NAME, program.created_at, mp4_path)
+ true
end
def title(program)
date = program.created_at.strftime('%Y_%m_%d')
- title = "#{date}_#{program.title}_#{program.comment}"
+ title = "#{date}_#{program.title}_#{program.episode_name}"
+ if program.cast.present?
+ cast = program.cast.gsub(',', ' ')
+ title += "_#{cast}"
+ end
+ title
end
end
end
View
@@ -1,119 +1,41 @@
require 'net/http'
require 'time'
require 'pp'
-require 'digest/md5'
-require 'moji'
module Hibiki
- class Program < Struct.new(:title, :comment, :rtmp_url)
- end
- class ProgramBase < Struct.new(:title, :comment, :short_name, :channel_id)
+ class Program < Struct.new(:access_id, :episode_id, :title, :episode_name, :cast)
end
class Scraping
def main
- (0..6).inject([]) do |program_list, wday|
- sleep(1)
- program_list += get_wday(wday)
+ get_list.reject do |program|
+ program.episode_id == nil
end
end
- def get_wday(wday)
- get_wday_base(wday).map do |base|
- sleep(1)
- add_program_detail(base)
- end.compact # remove nil
+ def get_list
+ programs = []
+ parsed = nil
+ page = 1
+ begin
+ uri = URI("https://vcms-api.hibiki-radio.jp/api/v1//programs?limit=8&page=#{page}")
+ res = Net::HTTP.get(uri)
+ raws = JSON.parse(res)
+ programs += raws.map{|raw| parse_program(raw) }
+ sleep 1
+ page += 1
+ end while raws.size == 8
+ programs
end
- def add_program_detail(base)
- rtmp_url = get_rtmp_url(base)
- unless rtmp_url
- return nil
- end
+ def parse_program(raw)
Program.new(
- base.title,
- base.comment,
- rtmp_url
+ raw['access_id'],
+ raw['latest_episode_id'],
+ raw['name'],
+ raw['latest_episode_name'],
+ raw['cast'],
)
end
-
- def get_wday_base(wday)
- dom_programs = get_wday_doms(wday)
- parse_programs(dom_programs)
- end
-
- def get_wday_doms(wday)
- uri = URI.parse("http://hibiki-radio.jp/get_program/#{wday}")
- html = Net::HTTP.get(uri)
- dom = Nokogiri::HTML.parse(html)
- dom.css('a')
- end
-
- def parse_programs(dom_programs)
- dom_programs.map do |dom_program|
- parse_program dom_program
- end
- end
-
- def parse_program(dom)
- title = parse_title(dom)
- comment = Moji.normalize_zen_han(dom.css('.hbkProgramComment').text)
- short_name, channel_id = parse_onclick(dom.attribute('onclick').value)
- ProgramBase.new(
- title,
- comment,
- short_name,
- channel_id
- )
- end
-
- def parse_title(dom)
- t = dom.css('.hbkProgramButton').text
- if t.blank?
- t = dom.css('.hbkProgramButtonNew').text
- end
-
- Moji.normalize_zen_han(t.strip.gsub(/(\r\n|\r|\n)/, ' '))
- end
-
- def parse_onclick(onclick_text)
- m = /AttachVideo\('(.+?)','(.+?)','.+?','.+?'\)/.match(onclick_text)
- [m[1], m[2]]
- end
-
- def get_rtmp_url(base)
- dom = get_channel_dom(base)
- unless dom
- return nil
- end
- parse_channel_dom(dom)
- end
-
- def parse_channel_dom(dom)
- protocol = dom.css('protocol').text
- domain = dom.css('domain').text
- dir = dom.css('dir').text
- flv = dom.css('flv').text
- if protocol.blank? || domain.blank? || dir.blank? || flv.blank?
- return nil
- end
- m = /^.+?\:(.+)$/.match(flv)
- filename_query = m[1]
- "#{protocol}://#{domain}/#{dir}/#{filename_query}"
- end
-
- def get_channel_dom(base)
- uri = URI.parse("http://image.hibiki-radio.jp/uploads/data/channel/#{base.short_name}/#{base.channel_id}.xml")
-
- res = Net::HTTP.get_response(uri)
- unless res.is_a?(Net::HTTPSuccess)
- return nil
- end
-
- # the response is XML but not valid.
- # parse as HTML to expect parsing more fuzzy.
- Nokogiri::HTML.parse(res.body)
- end
-
end
end
Oops, something went wrong.

0 comments on commit 8bb01a5

Please sign in to comment.