Skip to content
Permalink
Browse files

Support Hibiki renewal

  • Loading branch information
yayugu committed Nov 9, 2015
1 parent 9050217 commit 8bb01a577c1e9ad39da852502b620ea94d4f0b3a
@@ -26,20 +26,25 @@ Net Radio Archive
- LinuxなどUNIX的なOS (Windowsでも動かしたい...)
- Ruby 2.0 or higher
- rtmpdump
- ffmpeg or livav
- swftools
- あたらしめのffmpeg (HTTP Live Streaming の input に対応しているもの)
- ※最新のffmpegの導入は面倒であることが多いです。自分はLinuxではstatic buildを使っています。 http://qiita.com/yayugu/items/d7f6a15a6f988064f51c
- Macではhomebrewで導入できるバージョンで問題ありません
- (AG-ONのみ)
- GUI環境 or xvfb
- firefox
- とてもあたらしいffmpeg (HTTP Live Streaming の input に対応しているもの)
- ※最新のffmpegの導入は面倒であることが多いです。自分はLinux用のstatic buildを使っています。 http://qiita.com/yayugu/items/d7f6a15a6f988064f51c

## セットアップ

```
# 必要なライブラリをインストール
# Ubuntuの場合:
$ sudo apt-get install rtmpdump libav-tools swftools ruby
$ sudo apt-get install rtmpdump swftools ruby
$ # libavがインストールされている場合には削除してから
$ wget http://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz
$ tar xvf ffmpeg-release-64bit-static.tar.xz
$ sudo cp ./ffmpeg-release-64bit-static/ffmpeg /usr/local/bin
$ git clone https://github.com/yayugu/net-radio-archive.git
$ cd net-radio-archive
@@ -4,6 +4,8 @@ module OndemandRetry
downloading: 'downloading',
done: 'done',
failed: 'failed',
not_downloadable: 'not_downloadable',
outdated: 'outdated',
}
RETRY_LIMIT = 3
end
@@ -0,0 +1,3 @@
class HibikiProgramV2 < ActiveRecord::Base
include OndemandRetry
end
@@ -0,0 +1,25 @@
class CreateHibikiProgramV2 < ActiveRecord::Migration
def up
sql = <<EOF
CREATE TABLE `hibiki_program_v2s` (
`id` int UNSIGNED NOT NULL AUTO_INCREMENT,
`access_id` varchar(100) CHARACTER SET ASCII NOT NULL,
`episode_id` int UNSIGNED NOT NULL,
`title` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
`episode_name` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
`cast` varchar(250) CHARACTER SET utf8mb4 NOT NULL,
`state` varchar(100) CHARACTER SET ASCII NOT NULL,
`retry_count` int UNSIGNED NOT NULL,
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY (`access_id`, `episode_id`)
);
EOF
ActiveRecord::Base.connection.execute(sql)
end
def down
sql = 'DROP TABLE `hibiki_program_v2s`;'
ActiveRecord::Base.connection.execute(sql)
end
end
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema.define(version: 20151030150712) do
ActiveRecord::Schema.define(version: 20151110023306) do

create_table "agon_programs", force: true do |t|
t.string "title", limit: 250, null: false
@@ -39,6 +39,34 @@

add_index "anitama_programs", ["book_id", "update_time"], name: "book_id", unique: true, using: :btree

create_table "hibiki_program_v2", force: true do |t|
t.string "access_id", limit: 100, null: false
t.integer "episode_id", null: false
t.string "title", limit: 250, null: false
t.string "episode_name", limit: 250, null: false
t.string "cast", limit: 250, null: false
t.string "state", limit: 100, null: false
t.integer "retry_count", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end

add_index "hibiki_program_v2", ["access_id", "episode_id"], name: "access_id", unique: true, using: :btree

create_table "hibiki_program_v2s", force: true do |t|
t.string "access_id", limit: 100, null: false
t.integer "episode_id", null: false
t.string "title", limit: 250, null: false
t.string "episode_name", limit: 250, null: false
t.string "cast", limit: 250, null: false
t.string "state", limit: 100, null: false
t.integer "retry_count", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
end

add_index "hibiki_program_v2s", ["access_id", "episode_id"], name: "access_id", unique: true, using: :btree

create_table "hibiki_programs", force: true do |t|
t.string "title", limit: 250, null: false
t.string "comment", limit: 150, null: false
@@ -5,37 +5,71 @@ module Hibiki
class Downloading
CH_NAME = 'hibiki'

def initialize
@a = Mechanize.new
@a.user_agent_alias = 'Windows Chrome'
end

def download(program)
unless exec_rec(program)
return false
infos = get_infos(program)
if infos['episode']['id'] != program.episode_id
Rails.logger.error("episode outdated. title=#{program.title} expected_episode_id=#{program.episode_id} actual_episode_id=#{infos['episode']['id']}")
program.state = HibikiProgramV2::STATE[:outdated]
return
end
live_flg = infos['episode'].try(:[], 'video').try(:[], 'live_flg')
if live_flg == nil || live_flg == true
program.state = HibikiProgramV2::STATE[:not_downloadable]
return
end
exec_convert(program)
url = get_m3u8_url(infos['episode']['video']['id'])
unless download_hls(program, url)
program.state = HibikiProgramV2::STATE[:failed]
return
end
program.state = HibikiProgramV2::STATE[:done]
end

def exec_rec(program)
flv_path = Main::file_path_working(CH_NAME, title(program), 'flv')
command = "rtmpdump -q -r #{Shellwords.escape(program.rtmp_url)} -o #{Shellwords.escape(flv_path)}"
def get_infos(program)
res = @a.get("https://vcms-api.hibiki-radio.jp/api/v1/programs/#{program.access_id}")
infos = JSON.parse(res.body)
end

def get_m3u8_url(video_id)
res = @a.get("https://vcms-api.hibiki-radio.jp/api/v1/videos/play_check?video_id=#{video_id}")
play_infos = JSON.parse(res.body)
play_infos['playlist_url']
end

def download_hls(program, m3u8_url)
file_path = Main::file_path_working(CH_NAME, title(program), 'mp4')
arg = "\
-loglevel error \
-y \
-i #{Shellwords.escape(m3u8_url)} \
-vcodec copy -acodec copy -bsf:a aac_adtstoasc \
#{Shellwords.escape(file_path)}"

Main::prepare_working_dir(CH_NAME)
exit_status, output = Main::shell_exec(command)
exit_status, output = Main::ffmpeg(arg)
unless exit_status.success?
#Rails.logger.error "rec failed. program:#{program}, exit_status:#{exit_status}, output:#{output}"
Rails.logger.error "rec failed. program:#{program}, exit_status:#{exit_status}, output:#{output}"
return false
end

true
end
Main::move_to_archive_dir(CH_NAME, program.created_at, file_path)

def exec_convert(program)
flv_path = Main::file_path_working(CH_NAME, title(program), 'flv')
mp4_path = Main::file_path_working(CH_NAME, title(program), 'mp4')
Main::convert_ffmpeg_to_mp4(flv_path, mp4_path, program)
Main::move_to_archive_dir(CH_NAME, program.created_at, mp4_path)
true
end

def title(program)
date = program.created_at.strftime('%Y_%m_%d')
title = "#{date}_#{program.title}_#{program.comment}"
title = "#{date}_#{program.title}_#{program.episode_name}"
if program.cast.present?
cast = program.cast.gsub(',', ' ')
title += "_#{cast}"
end
title
end
end
end
@@ -1,119 +1,41 @@
require 'net/http'
require 'time'
require 'pp'
require 'digest/md5'
require 'moji'

module Hibiki
class Program < Struct.new(:title, :comment, :rtmp_url)
end
class ProgramBase < Struct.new(:title, :comment, :short_name, :channel_id)
class Program < Struct.new(:access_id, :episode_id, :title, :episode_name, :cast)
end

class Scraping
def main
(0..6).inject([]) do |program_list, wday|
sleep(1)
program_list += get_wday(wday)
get_list.reject do |program|
program.episode_id == nil
end
end

def get_wday(wday)
get_wday_base(wday).map do |base|
sleep(1)
add_program_detail(base)
end.compact # remove nil
def get_list
programs = []
parsed = nil
page = 1
begin
uri = URI("https://vcms-api.hibiki-radio.jp/api/v1//programs?limit=8&page=#{page}")
res = Net::HTTP.get(uri)
raws = JSON.parse(res)
programs += raws.map{|raw| parse_program(raw) }
sleep 1
page += 1
end while raws.size == 8
programs
end

def add_program_detail(base)
rtmp_url = get_rtmp_url(base)
unless rtmp_url
return nil
end
def parse_program(raw)
Program.new(
base.title,
base.comment,
rtmp_url
raw['access_id'],
raw['latest_episode_id'],
raw['name'],
raw['latest_episode_name'],
raw['cast'],
)
end

def get_wday_base(wday)
dom_programs = get_wday_doms(wday)
parse_programs(dom_programs)
end

def get_wday_doms(wday)
uri = URI.parse("http://hibiki-radio.jp/get_program/#{wday}")
html = Net::HTTP.get(uri)
dom = Nokogiri::HTML.parse(html)
dom.css('a')
end

def parse_programs(dom_programs)
dom_programs.map do |dom_program|
parse_program dom_program
end
end

def parse_program(dom)
title = parse_title(dom)
comment = Moji.normalize_zen_han(dom.css('.hbkProgramComment').text)
short_name, channel_id = parse_onclick(dom.attribute('onclick').value)
ProgramBase.new(
title,
comment,
short_name,
channel_id
)
end

def parse_title(dom)
t = dom.css('.hbkProgramButton').text
if t.blank?
t = dom.css('.hbkProgramButtonNew').text
end

Moji.normalize_zen_han(t.strip.gsub(/(\r\n|\r|\n)/, ' '))
end

def parse_onclick(onclick_text)
m = /AttachVideo\('(.+?)','(.+?)','.+?','.+?'\)/.match(onclick_text)
[m[1], m[2]]
end

def get_rtmp_url(base)
dom = get_channel_dom(base)
unless dom
return nil
end
parse_channel_dom(dom)
end

def parse_channel_dom(dom)
protocol = dom.css('protocol').text
domain = dom.css('domain').text
dir = dom.css('dir').text
flv = dom.css('flv').text
if protocol.blank? || domain.blank? || dir.blank? || flv.blank?
return nil
end
m = /^.+?\:(.+)$/.match(flv)
filename_query = m[1]
"#{protocol}://#{domain}/#{dir}/#{filename_query}"
end

def get_channel_dom(base)
uri = URI.parse("http://image.hibiki-radio.jp/uploads/data/channel/#{base.short_name}/#{base.channel_id}.xml")

res = Net::HTTP.get_response(uri)
unless res.is_a?(Net::HTTPSuccess)
return nil
end

# the response is XML but not valid.
# parse as HTML to expect parsing more fuzzy.
Nokogiri::HTML.parse(res.body)
end

end
end

0 comments on commit 8bb01a5

Please sign in to comment.
You can’t perform that action at this time.