Permalink
Browse files

Upgraded download and extract engine

I spent quite a bit of time upgrading the extract engine. It now return all the formats, in a vector (including the real_urls)
It has a few validation checks before it tried to grab the format and real_url.
I've also added a wxMsg to current_videos.extension in MainFrame.cpp, incase you have to login to view the video, or that it is blocked by region.
My next push should contain comments, as I have not put them in yet.
  • Loading branch information...
1 parent 3872b3b commit c655fd3af7f0671ec05ecdcf830694317b7dbf01 Afterburn committed Jun 26, 2011
Showing with 188 additions and 47 deletions.
  1. +77 −20 src/Extract.cpp
  2. +10 −1 src/Extract.hpp
  3. +1 −0 src/MainFrame.cpp
  4. +2 −2 src/Makefile
  5. +0 −22 src/pymake
  6. +96 −0 tests/download_prototype/iter_formats.cxx
  7. +2 −2 tests/download_prototype/pymake
View
@@ -2,6 +2,7 @@
#include <cstdlib>
#include "curl/curl.h"
#include <boost/format.hpp>
+#include <boost/algorithm/string/split.hpp>
#include <sstream>
#include <string>
#include "urilite.h"
@@ -10,6 +11,12 @@
std::string Extract::resolve_buffer;
+std::vector<std::string> Extract::formats;
+std::string Extract::mp4_url;
+std::string Extract::flv_url;
+std::string Extract::three_gp_url;
+
+
int start;
int Extract::writer(char *data, size_t size, size_t nmemb, std::string *resolve_buffer)
@@ -47,38 +54,88 @@ std::string Extract::format_url(std::string id)
}
-std::string Extract::extension()
+
+
+
+void Extract::gather_formats()
{
+
+ using namespace boost;
std::string format;
+
start = resolve_buffer.find("fmt_url_map=");
start = start + 12;
- switch (atoi(resolve_buffer.substr(start, 2).c_str()))
+ //std::vector<std::string> formats;
+ std::vector<std::string> tmp;
+ std::string fmt_map = urilite::uri::decode2(resolve_buffer.substr(start, resolve_buffer.length()).c_str());
+ split(tmp, fmt_map, is_any_of(","));
+
+
+ for (int num = 0; num < tmp.size(); num++)
{
- case 5 :
- case 34:
- case 35:
- format = "flv";
- break;
+ std::string tmp_str = tmp[num].substr(0,3);
- case 18:
- case 22:
- case 37:
- case 38:
- format = "mp4";
- break;
+ if (tmp_str.find("|") == std::string::npos)
+ {
+ //std::cout << "Not valid" << std::endl;
+ continue;
+ }
+
+ else
+ {
+ std::string real_url = tmp[num].substr(tmp[num].find("http:"), tmp[num].length());
+
+ if (tmp[num].find("http:") != std::string::npos)
+ {
+ switch (atoi(tmp_str.c_str()))
+ {
+ case 5 :
+ case 34:
+ case 35:
+ formats.push_back("flv");
+ flv_url = real_url;
+ break;
+
+ case 18:
+ case 22:
+ case 37:
+ case 38:
+ formats.push_back("mp4");
+ mp4_url = real_url;
+ break;
- case 17:
- format = "3gp";
- break;
+ case 17:
+ formats.push_back("3gp");
+ three_gp_url = real_url;
+ break;
- default:
- format = "";
- break;
+ default:
+ std::cout << "No format found" << std::endl;
+ break;
+
+ }
+ }
+ }
}
- return format;
}
+
+std::string Extract::extension()
+{
+ std::string best_format = "";
+ gather_formats();
+
+ if (mp4_url != "") { return "mp4"; }
+ if (flv_url != "") { return "flv"; }
+ if (three_gp_url != "") { return "3gp"; }
+
+
+
+ return best_format;
+}
+
+
std::string Extract::resolve_real_url(std::string id)
{
//This function connects to the internet, and retrieves the
View
@@ -1,20 +1,29 @@
#include <iostream>
+#include <vector>
//I have not read too much about making .hpp files, so correct this as you see fit.
class Extract
{
public:
+ static std::string mp4_url;
+ static std::string flv_url;
+ static std::string three_gp_url;
static std::string resolve_real_url(std::string id);
static std::string format_url(std::string id);
static std::string resolve_buffer;
static std::string extension();
+ static void gather_formats();
+ static std::vector<std::string> formats;
+
+
+
struct video_struct
{
std::string actual_url;
std::string id;
- std::string format;
+ std::string best_ext;
std::string title;
std::string save_dir;
std::string extension;
View
@@ -240,6 +240,7 @@ void MainFrame::OnVideoDownload(wxCommandEvent& WXUNUSED(event))
current_video.title = info->getName();
current_video.save_dir = path_dlg.GetDirectory().mb_str();
current_video.extension = Extract::extension();
+ if (current_video.extension == "") { wxMessageBox(_("We could not retrieve the download link for this video, this can be caused by the video being 18+ (requires login) or the video is not available in your country. We are implementing this feature, hang tight."), _("Our apologies"), wxOK | wxICON_INFORMATION); return;}
current_video.full_save_path = std::string(
path_dlg.GetPath().mb_str()).append(".").append(current_video.extension);
View
@@ -4,8 +4,8 @@ LIBS=`wx-config --cxxflags --libs`
all: wxyoutube
-wxyoutube: Enums.hpp ProgressBar.o Parser.o ImageProc.o ImageBox.o ThumbnailFrame.o VideoInfo.o VideoDescription.o EventHandler.hpp Events.o EventManager.o DownloadCallback.o SearchURL.o XMLFeed.o DownloadThread.o VideoEntry.o PrefWindow.o VideoListCtrl.o MainFrame.o MainApp.o
- $(CC) -o Youtube-wx Enums.hpp ProgressBar.o Parser.o ImageProc.o ImageBox.o ThumbnailFrame.o VideoInfo.o VideoDescription.o EventHandler.hpp Events.o EventManager.o DownloadCallback.o SearchURL.o XMLFeed.o DownloadThread.o VideoEntry.o PrefWindow.o VideoListCtrl.o MainFrame.o MainApp.o Extract.hpp Extract.cpp $(CFLAGS) $(LIBS)
+wxyoutube: Enums.hpp ProgressBar.o Parser.o ImageProc.o ImageBox.o ThumbnailFrame.o VideoInfo.o VideoDescription.o EventHandler.hpp Events.o EventManager.o DownloadCallback.o SearchURL.o XMLFeed.o DownloadThread.o VideoEntry.o PrefWindow.o VideoListCtrl.o MainFrame.o MainApp.o Extract.o
+ $(CC) -o Youtube-wx Enums.hpp ProgressBar.o Parser.o ImageProc.o ImageBox.o ThumbnailFrame.o VideoInfo.o VideoDescription.o EventHandler.hpp Events.o EventManager.o DownloadCallback.o SearchURL.o XMLFeed.o DownloadThread.o VideoEntry.o PrefWindow.o VideoListCtrl.o MainFrame.o MainApp.o Extract.o $(CFLAGS) $(LIBS)
XMLFeed.o: XMLFeed.hpp XMLFeed.cpp
$(CC) -c -o XMLFeed.o XMLFeed.cpp $(CFLAGS) $(LIBS)
View
@@ -1,22 +0,0 @@
-#!/usr/bin/env python2
-
-from os import system
-
-#File to compile
-#Output file options
-output_name="Youtube-wx"
-
-files = "VideoSearch.hpp VideoSearch.cpp Enums.hpp VideoInfo.hpp VideoInfo.cpp SearchURL.hpp SearchURL.cpp DownloadThread.hpp DownloadThread.cpp VideoEntry.hpp VideoEntry.cpp VideoListCtrl.hpp VideoListCtrl.cpp PrefWindow.hpp PrefWindow.cpp MainFrame.hpp MainFrame.cpp MainApp.hpp MainApp.cpp"
-
-#Compiler args
-args=" -lcurl `wx-config --cxxflags --libs`"
-
-#Compiler
-compiler="g++"
-
-command = "%s %s -o %s %s" % (compiler, files, output_name, args)
-
-print "Using command: ", command
-
-system(command)
-
@@ -0,0 +1,96 @@
+#include <iostream>
+#include <string>
+#include "urilite.h"
+#include <cstring>
+#include <boost/algorithm/string/split.hpp>
+#include <vector>
+
+
+
+
+std::vector<std::string> gather_formats(std::string format_map)
+{
+ using namespace boost;
+ std::string format;
+
+ std::string mp4_url;
+ std::string flv_url;
+ std::string three_gp_url;
+
+ std::vector<std::string> formats;
+ std::vector<std::string> tmp;
+ split(tmp, format_map, is_any_of(","));
+
+
+
+ for (int num = 0; num < tmp.size(); num++)
+ {
+
+
+
+ std::string tmp_str = tmp[num].substr(0,3);
+
+ if (tmp_str.find("|") == std::string::npos)
+ {
+ //std::cout << "Not valid" << std::endl;
+ continue;
+ }
+
+ else
+ {
+ std::string real_url = tmp[num].substr(tmp[num].find("http:"), tmp[num].length());
+
+ if (tmp[num].find("http:") != std::string::npos)
+ {
+
+
+ switch (atoi(tmp_str.c_str()))
+ {
+ case 5 :
+ case 34:
+ case 35:
+ formats.push_back("flv");
+ flv_url = real_url;
+ break;
+
+ case 18:
+ case 22:
+ case 37:
+ case 38:
+ formats.push_back("mp4");
+ mp4_url = real_url;
+ break;
+
+ case 17:
+ formats.push_back("3gp");
+ three_gp_url = real_url;
+ break;
+
+ default:
+ std::cout << "No format found" << std::endl;
+ break;
+
+ }
+ }
+ }
+ }
+ return formats;
+
+}
+
+int main()
+{
+
+
+
+ std::string fmt_map = "fmt_url_map=34%7Chttp%3A%2F%2Fv8.lscache2.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D34%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3DAA5A697A787F3D3C657C502552DD2C4418F30A54.5082EB2362FCEE56EE0CE27AFE7CF769493A92AC%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4%2C18%7Chttp%3A%2F%2Fv18.lscache3.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D18%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3D0FC5F2BD058888EE54670E65FD00C954DC56467F.39CF7A23F5352F5FDD83F4453DC8A9AD486C139A%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4%2C5%7Chttp%3A%2F%2Fv7.lscache2.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D5%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3D99B6284F841ABA5CB15B234516CC836B51FF8B60.A376C1727ED61B38E6EDBD45E4578ECB071E249A%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4&allow_ratings=1&keywords=WaT%2CEiji%2CWentz%2CTeppei%2CKoike%2C5%E3%82%BB%E3%83%B3%E3%83%81%2CGo%2CSenchi%2C5cm%2CMV%2CJ-pop%2CJ-Rock&track_embed=0&view_count=98617&fmt_list=34%2F320x240%2F9%2F0%2F115%2C18%2F320x240%2F9%2F0%2F115%2C5%2F320x240%2F7%2F0%2F0&author=CaroYuuki&muted=0&length_seconds=321&user_age=30&plid=AASmiya2oal4_cmk&ftoken=DCvtsH2pCH51yKPbdzjJ7VDfevd8MTMwOTEwMjg4N0AxMzA5MDE2NDg3&status=ok&watermark=http%3A%2F%2Fs.ytimg.com%2Fyt%2Fswf%2Flogo-vfl_bP6ud.swf%2Chttp%3A%2F%2Fs.ytimg.com%2Fyt%2Fswf%2Fhdlogo-vfloR6wva.swf&timestamp=1309016487&has_cc=False&watch_ajax_token=k_Frd8rvbGHrSCmod3IVHEVR0YR8MTMwOTEwMjg4N0AxMzA5MDE2NDg3&fmt_map=34%2F320x240%2F9%2F0%2F115%2C18%2F320x240%2F9%2F0%2F115%2C5%2F320x240%2F7%2F0%2F0&leanback_module=http%3A%2F%2Fs.ytimg.com%2Fyt%2Fswfbin%2Fleanback_module-vflNXuMTT.swf&hl=en_US&endscreen_module=http%3A%2F%2Fs.ytimg.com%2Fyt%2Fswfbin%2Fendscreen-vfls7ND3M.swf&fmt_stream_map=34%7Chttp%3A%2F%2Fv8.lscache2.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D34%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3DAA5A697A787F3D3C657C502552DD2C4418F30A54.5082EB2362FCEE56EE0CE27AFE7CF769493A92AC%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4%7C%7Ctc.v8.cache2.c.youtube.com%2C18%7Chttp%3A%2F%2Fv18.lscache3.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D18%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3D0FC5F2BD058888EE54670E65FD00C954DC56467F.39CF7A23F5352F5FDD83F4453DC8A9AD486C139A%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4%7C%7Ctc.v18.cache3.c.youtube.com%2C5%7Chttp%3A%2F%2Fv7.lscache2.c.youtube.com%2Fvideoplayback%3Fsparams%3Did%252Cexpire%252Cip%252Cipbits%252Citag%252Calgorithm%252Cburst%252Cfactor%252Coc%253AU0hPTlBOV19FSkNOOV9ISEFD%26fexp%3D903929%252C907605%252C907321%26algorithm%3Dthrottle-factor%26itag%3D5%26ip%3D0.0.0.0%26burst%3D40%26sver%3D3%26signature%3D99B6284F841ABA5CB15B234516CC836B51FF8B60.A376C1727ED61B38E6EDBD45E4578ECB071E249A%26expire%3D1309039200%26key%3Dyt1%26ipbits%3D0%26factor%3D1.25%26id%3D38db2ffda2b671f4%7C%7Ctc.v7.cache2.c.youtube.com&avg_rating=5.0&logwatch=1&vq=auto&token=vjVQa1PpcFM0SfYbF6BS7TC1LF385XZZKyGWlcwNcQ0%3D&thumbnail_url=http%3A%2F%2Fi4.ytimg.com%2Fvi%2FONsv_aK2cfQ%2Fdefault.jpg&video_id=ONsv_aK2cfQ&title=WaT+-+5%E3%82%BB%E3%83%B3%E3%83%81+%28Go+Senchi+%2F+5+cm%29+MV";
+
+ std::vector<std::string> fmts = gather_formats(urilite::uri::decode2(fmt_map));
+ //for (int v = 0; v != fmts.size(); v++) { std::cout << fmts[v] << std::endl; }
+
+
+ return 0;
+
+}
+
+
@@ -6,10 +6,10 @@ from os import system
#Output file options
output_name="downloader"
-files = "download.cxx"
+files = "iter_formats.cxx"
#Compiler args
-args=" -I/usr/include/QtCore -I/usr/include/QtGui -I/usr/include -I. -lcurl `wx-config --cxxflags --libs`"
+args=" -I/usr/include -I. -lcurl `wx-config --cxxflags --libs`"
#Compiler
compiler="g++"

0 comments on commit c655fd3

Please sign in to comment.