Permalink
Browse files

Added Podnapisi.NET support

  • Loading branch information...
1 parent 60025ec commit 878d3fa8a8c6ff0388e6f2bf2fd1093f64674e59 @blaztinn committed Aug 29, 2011
Showing with 425 additions and 12 deletions.
  1. +4 −2 README.asciidoc
  2. +2 −2 configure.ac
  3. +12 −2 doc/README.html
  4. +5 −4 lib/Makefile.am
  5. +327 −0 lib/podnapisi.vala
  6. +75 −2 lib/server.vala
View
6 README.asciidoc
@@ -1,7 +1,7 @@
Submarine
===========
Blaž Tomažič <blaz.tomazic@gmail.com>
-0.1.0, 2011-08-11
+0.1.0, 2011-08-29
:Author: Blaž Tomažič
:Email: blaz.tomazic@gmail.com
:Date: 2011-08-11
@@ -73,6 +73,7 @@ Submarine has little dependencies and is really lightweight.
- glib2
- libgee
- libsoup
+- libarchive
.Build
- vala
@@ -107,4 +108,5 @@ link="https://flattr.com/submit/auto?user_id=blazt&url=http://github.com/blazt/s
== Powered by
This application is based on the following online subtitle services:
-- *OpenSubtitles* (http://www.opensubtitles.org)
+- *OpenSubtitles.org* (http://www.opensubtitles.org)
+- *Podnapisi.NET* (http://www.opensubtitles.org)
View
4 configure.ac
@@ -16,11 +16,11 @@ AM_PROG_CC_C_O
AM_PROG_VALAC
# Checks for libraries.
-libsubmarine_modules="glib-2.0 gobject-2.0 libsoup-2.4 gee-1.0"
+libsubmarine_modules="glib-2.0 gobject-2.0 libsoup-2.4 gee-1.0 libarchive"
PKG_CHECK_MODULES([LIBSUBMARINE], [$libsubmarine_modules])
AC_SUBST([LIBSUBMARINE_CFLAGS])
AC_SUBST([LIBSUBMARINE_LIBS])
-LIBSUBMARINE_VALAPACKAGES="--pkg gee-1.0 --pkg libsoup-2.4"
+LIBSUBMARINE_VALAPACKAGES="--pkg gee-1.0 --pkg libsoup-2.4 --pkg libarchive"
AC_SUBST([LIBSUBMARINE_VALAPACKAGES])
submarine_modules="$libsubmarine_modules gio-2.0"
View
14 doc/README.html
@@ -841,6 +841,11 @@ <h3 id="_dependecies">Dependecies</h3>
libsoup
</p>
</li>
+<li>
+<p>
+libarchive
+</p>
+</li>
</ul></div>
<div class="ulist"><div class="title">Build</div><ul>
<li>
@@ -894,7 +899,12 @@ <h2 id="_powered_by">Powered by</h2>
<div class="ulist"><ul>
<li>
<p>
-<strong>OpenSubtitles</strong> (<a href="http://www.opensubtitles.org">http://www.opensubtitles.org</a>)
+<strong>OpenSubtitles.org</strong> (<a href="http://www.opensubtitles.org">http://www.opensubtitles.org</a>)
+</p>
+</li>
+<li>
+<p>
+<strong>Podnapisi.NET</strong> (<a href="http://www.opensubtitles.org">http://www.opensubtitles.org</a>)
</p>
</li>
</ul></div>
@@ -905,7 +915,7 @@ <h2 id="_powered_by">Powered by</h2>
<div id="footer">
<div id="footer-text">
Version 0.1.0<br />
-Last updated 2011-08-11 13:44:29 CEST
+Last updated 2011-08-29 18:25:43 CEST
</div>
</div>
</body>
View
9 lib/Makefile.am
@@ -1,10 +1,11 @@
noinst_LTLIBRARIES = libsubmarine.la
libsubmarine_la_SOURCES = language_codes.vala \
- subtitle.vala \
- server.vala \
- open_subtitles.vala \
- session.vala
+ subtitle.vala \
+ server.vala \
+ open_subtitles.vala \
+ podnapisi.vala \
+ session.vala
libsubmarine_la_VALAFLAGS = $(LIBSUBMARINE_VALAPACKAGES) --library submarine -H submarine.h
libsubmarine_la_CPPFLAGS = $(LIBSUBMARINE_CFLAGS)
View
327 lib/podnapisi.vala
@@ -0,0 +1,327 @@
+namespace Submarine {
+
+ private class PodnapisiServer : SubtitleServer {
+ private Soup.SessionSync session;
+ private string session_token;
+
+ private Gee.HashSet<string> supported_languages;
+ private Gee.HashMap<string, int> language_ids;
+ private Gee.HashSet<string> selected_languages;
+
+ construct {
+ this.info = ServerInfo("Podnapisi",
+ "http://www.podnapisi.net/",
+ "pn");
+ }
+
+ private bool filter_languages(Gee.Collection<string> languages) {
+ if(this.supported_languages == null) {
+ this.supported_languages = new Gee.HashSet<string>();
+ this.language_ids = new Gee.HashMap<string, int>();
+
+ var message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2",
+ "supportedLanguages",
+ typeof(string), this.session_token);
+
+ if(this.session.send_message(message) == 200) {
+ try {
+ Value v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v);
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)v;
+
+ if((int)(vh.lookup("status")) == 200) {
+ unowned ValueArray va = (ValueArray) vh.lookup("languages");
+
+ foreach(Value vresult in va) {
+ this.supported_languages.add((string)((ValueArray)vresult).get_nth(1));
+ this.language_ids.set((string)((ValueArray)vresult).get_nth(1),
+ (int)((ValueArray)vresult).get_nth(0));
+ }
+ }
+ } catch(Error e) {}
+ }
+ }
+
+ var languages_set = new Gee.HashSet<string>();
+
+ foreach(var language in languages) {
+ var language_info = get_language_info(language);
+
+ //podnapisi.net supports only short codes
+ if(language_info.short_code != null && language_info.short_code in supported_languages) {
+ languages_set.add(language_info.short_code);
+ }
+ }
+
+ if(!languages_set.is_empty) {
+ var update = false;
+ if(this.selected_languages == null || languages_set.size != this.selected_languages.size) {
+ update = true;
+ } else {
+ foreach(var language in languages_set) {
+ if(!(language in this.selected_languages)) {
+ update = true;
+ break;
+ }
+ }
+ }
+
+ if(update) {
+ var languages_array = new ValueArray(0);
+ foreach(var language in languages_set) {
+ languages_array.append(this.language_ids[language]);
+ }
+
+ var message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2", //TODO: save session languages to some variable and set only once
+ "setFilters",
+ typeof(string), this.session_token,
+ typeof(bool), true,
+ typeof(ValueArray), languages_array,
+ typeof(bool), false);
+
+ try {
+ if(this.session.send_message(message) == 200) {
+ Value v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v); //TODO: na tu dej tudi IF(...)!
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)v;
+
+ if((int)(vh.lookup("status")) == 200) {
+ this.selected_languages = languages_set;
+ return true;
+ }
+ }
+ } catch (Error r) {}
+ } else {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private uint64 file_size(File file) throws Error {
+ var file_info = file.query_info("*", FileQueryInfoFlags.NONE);
+ return file_info.get_size();
+ }
+
+ private uint64 file_hash(File file) throws Error,IOError {
+ uint64 hash, size;
+
+ //get filesize and add it to hash
+ size = this.file_size(file);
+ hash = size;
+
+ //add first 64kB of file to hash
+ var dis = new DataInputStream(file.read());
+ dis.set_byte_order(DataStreamByteOrder.LITTLE_ENDIAN);
+ for(int i=0; i<65536/sizeof(uint64); i++) {
+ hash += dis.read_uint64();
+ }
+ //add last 64kB of file to hash
+ dis = new DataInputStream(file.read());
+ dis.set_byte_order(DataStreamByteOrder.LITTLE_ENDIAN);
+ dis.skip((size_t)(size - 65536));
+ for(int i=0; i<65536/sizeof(uint64); i++) {
+ hash += dis.read_uint64();
+ }
+
+ return hash;
+ }
+
+ private bool inflate_subtitle(uint8[] data, out string format, out string inflated_data) {
+ var archive = new Archive.Read();
+
+ archive.support_format_zip();
+
+ if(archive.open_memory(data, data.length) == Archive.Result.OK) {
+ weak Archive.Entry e;
+
+ while(archive.next_header(out e) == Archive.Result.OK) {
+ if (!Posix.S_ISDIR(e.mode())) {
+ inflated_data = string.nfill((size_t)e.size(), ' ');
+ archive.read_data(inflated_data.data, inflated_data.data.length);
+
+ format = e.pathname().substring(e.pathname().last_index_of(".")+1);
+
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ public override bool connect() {
+ const string username = "submarine";
+ const string password = "password";
+ this.session = new Soup.SessionSync();
+
+ var message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2",
+ "initiate",
+ typeof(string), "submarine");
+
+ if(this.session.send_message(message) == 200) {
+ try {
+ Value v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v);
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)v;
+
+
+ if((int)(vh.lookup("status")) == 200) {
+ var nonce = (string)(vh.lookup("nonce"));
+ this.session_token = (string)(vh.lookup("session"));
+
+ var formatted_password = Checksum.compute_for_string(ChecksumType.SHA256,
+ Checksum.compute_for_string(ChecksumType.MD5, password) + nonce);
+
+ message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2",
+ "authenticate",
+ typeof(string), this.session_token,
+ typeof(string), username,
+ typeof(string), formatted_password);
+
+ if(this.session.send_message(message) == 200) {
+ v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v);
+ vh = (HashTable<string,Value?>)v;
+
+ if((int)(vh.lookup("status")) == 200) {
+ return true;
+ }
+ }
+ }
+ } catch (Error e) {}
+ }
+
+ return false;
+ }
+
+ public override void disconnect() {
+ }
+
+ //Note: search() not implemented, because there is minimal improvement over search_multiple()
+
+ public override Gee.MultiMap<File, Subtitle> search_multiple(Gee.Collection<File> files, Gee.Collection<string> languages) {
+ var subtitles_found_map = new Gee.HashMultiMap<File, Subtitle>();
+ var requests = new Gee.ArrayList<Value?>();
+ var hash_file = new Gee.HashMap<string, File>();
+
+ //maximum response results is 500
+ const int MAX = 500;
+ //assume 5 hits per subtitle
+ const int HITS = 5;
+
+ if(this.filter_languages(languages)) {
+ foreach (var file in files) {
+ try {
+ string hash = "%016llx".printf(this.file_hash(file));
+
+ requests.add(hash);
+
+ hash_file.set(hash, file);
+ } catch(Error e) {}
+ }
+
+ SubtitleServer.BatchRequestMethod request_method = (request_batch) => {
+ var values = new ValueArray(request_batch.size);
+
+ foreach(var request in request_batch) {
+ values.append(request);
+ }
+
+ var message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2",
+ "search",
+ typeof(string), this.session_token,
+ typeof(ValueArray), values);
+
+ try {
+ if(this.session.send_message(message) == 200) {
+ Value v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v);
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)v;
+
+ if((int)(vh.lookup("status")) == 200 && ((HashTable<string,Value?>)vh.lookup("results")).size() > 0) {
+ return v;
+ }
+ }
+ } catch(Error e) {}
+
+ return null;
+ };
+
+ SubtitleServer.BatchResponseMethod response_method = (response) => {
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)response;
+ int results = 0;
+
+ if((int)(vh.lookup("status")) == 200) {
+ foreach(string hash in ((HashTable<string,Value?>)vh.lookup("results")).get_keys()) {
+ unowned ValueArray va = (ValueArray) ((HashTable<string,Value?>)((HashTable<string,Value?>)vh.lookup("results")).lookup(hash)).lookup("subtitles");
+
+ foreach(Value vresult in va) {
+ HashTable<string,Value?> result = (HashTable<string,Value?>)vresult;
+
+ Subtitle subtitle = new Subtitle(this.info, result);
+ subtitle.format = "";
+ subtitle.language = (string)result.lookup("lang");
+ subtitle.rating = (int)result.lookup("rating")*2;
+
+ subtitles_found_map.set(hash_file[hash], subtitle);
+
+ results++;
+ }
+ }
+ }
+
+ return results;
+ };
+
+ this.batch_request("SearchSubtitles", requests, request_method, response_method, MAX/HITS, MAX);
+ }
+
+ return subtitles_found_map;
+ }
+
+ public override Subtitle? download(Subtitle subtitle) {
+ var requests = new ValueArray(0);
+
+ HashTable<string,Value?> server_data = (HashTable<string,Value?>)subtitle.server_data;
+ requests.append(server_data.lookup("id"));
+
+ var message = Soup.XMLRPC.request_new ("http://ssp.podnapisi.net:8000/RPC2",
+ "download",
+ typeof(string), this.session_token,
+ typeof(ValueArray), requests);
+
+ if(this.session.send_message(message) == 200) {
+ try {
+ Value v = Value (typeof (HashTable<string,Value?>));
+ Soup.XMLRPC.parse_method_response ((string) message.response_body.flatten().data, -1, v);
+ HashTable<string,Value?> vh = (HashTable<string,Value?>)v;
+
+ if((int)(vh.lookup("status")) == 200 && ((ValueArray)vh.lookup("names")).get_nth(0) != null) {
+ HashTable<string,Value?> result = (HashTable<string,Value?>)((ValueArray)vh.lookup("names")).get_nth(0);
+
+ message = new Soup.Message("GET", "http://www.podnapisi.net/static/podnapisi/" + (string)result.lookup("filename"));
+
+ if(this.session.send_message(message) == 200) {
+ string data;
+ string format;
+
+ if(this.inflate_subtitle(message.response_body.data, out format, out data)) {
+ subtitle.data = data;
+ subtitle.format = format;
+
+ return subtitle;
+ }
+ }
+ }
+ } catch (Error e) {}
+ }
+
+ return null;
+ }
+
+ //Note: download_multiple() not implemented, because there is no improvement over download()
+ }
+
+}
View
77 lib/server.vala
@@ -28,8 +28,22 @@ namespace Submarine {
public abstract new bool connect();
public abstract new void disconnect();
- public abstract Gee.Set<Subtitle> search(File file, Gee.Collection<string> languages);
- public abstract Subtitle? download(Subtitle subtitle);
+
+ //Note: Children of this class must override at least one of the next two functions!
+ public virtual Gee.Set<Subtitle> search(File file, Gee.Collection<string> languages) {
+ var subtitles_found = new Gee.HashSet<Subtitle>();
+
+ var files = new Gee.HashSet<File>();
+ files.add(file);
+
+ var subtitles_found_map = this.search_multiple(files, languages);
+
+ if(file in subtitles_found_map) {
+ subtitles_found.add_all(subtitles_found_map[file]);
+ }
+
+ return subtitles_found;
+ }
public virtual Gee.MultiMap<File, Subtitle> search_multiple(Gee.Collection<File> files, Gee.Collection<string> languages) {
var subtitles_found_map = new Gee.HashMultiMap<File, Subtitle>();
@@ -43,6 +57,22 @@ namespace Submarine {
return subtitles_found_map;
}
+ //Note: Children of this class must override at least one of the next two functions!
+ public virtual Subtitle? download(Subtitle subtitle) {
+ var subtitles = new Gee.HashSet<Subtitle>();
+ subtitles.add(subtitle);
+
+ var subtitles_downloaded = this.download_multiple(subtitles);
+
+ if(!subtitles_downloaded.is_empty) {
+ var it = subtitles_downloaded.iterator();
+ it.next();
+ return it.get();
+ }
+
+ return null;
+ }
+
public virtual Gee.Set<Subtitle> download_multiple(Gee.Collection<Subtitle> subtitles) {
var subtitles_downloaded = new Gee.HashSet<Subtitle>();
@@ -55,6 +85,48 @@ namespace Submarine {
return subtitles_downloaded;
}
+
+ protected delegate Value? BatchRequestMethod(Gee.List<Value?> request_batch);
+ protected delegate int BatchResponseMethod(Value response);
+ protected Gee.ArrayList<Value?> batch_request(string function_name, Gee.List<Value?> requests, BatchRequestMethod request_method, BatchResponseMethod response_method, int max_request_size, int max_response_size = 0)
+ requires (max_request_size > 0)
+ requires (max_response_size >= 0)
+ {
+ var batch_size = max_request_size;
+ var responses = new Gee.ArrayList<Value?>();
+ max_response_size = max_response_size > 0 ? max_response_size : max_request_size;
+
+ var request_index = 0;
+ while(request_index < requests.size) {
+ batch_size = requests.size-request_index < batch_size ? requests.size-request_index : batch_size;
+ var requests_batch = new Gee.ArrayList<Value?>();
+
+ for(int i = request_index; i < request_index+batch_size; i++) {
+ requests_batch.add(requests[i]);
+ }
+
+ Value? response = request_method(requests);
+
+ bool advance = true;
+ if(response != null) {
+ int results = response_method(response);
+
+ if(results < max_response_size || batch_size == 1) {
+ responses.add(response);
+ } else {
+ batch_size /= 2;
+ batch_size = batch_size > 0 ? batch_size : 1;
+ advance = false;
+ }
+ }
+
+ if(advance) {
+ request_index += batch_size;
+ }
+ }
+
+ return responses;
+ }
}
private Gee.List<string> all_server_codes = null;
@@ -64,6 +136,7 @@ namespace Submarine {
var all_servers = new Gee.HashSet<SubtitleServer>();
all_servers.add(new OpenSubtitlesServer());
+ all_servers.add(new PodnapisiServer());
return all_servers.read_only_view;
}

0 comments on commit 878d3fa

Please sign in to comment.