Permalink
Browse files

Initial import

This comes from my own private repository. This one is public.
  • Loading branch information...
0 parents commit a2107cdfda9088b02f6eb23deeafc427e503ad1a @yannk committed Mar 9, 2010
Showing with 8,568 additions and 0 deletions.
  1. +23 −0 README
  2. +3 −0 SETUP
  3. +18 −0 bin/add_user.pl
  4. +37 −0 bin/admin.pl
  5. +24 −0 bin/backup.sh
  6. +101 −0 bin/catchup_fetch.pl
  7. +9 −0 bin/debug
  8. +19 −0 bin/delete_user.pl
  9. +9 −0 bin/extract_meta_from_uri
  10. +146 −0 bin/failed_sched.pl
  11. +26 −0 bin/fetch.pl
  12. +43 −0 bin/fetch_all.pl
  13. +23 −0 bin/fetch_task.pl
  14. +20 −0 bin/fetch_tweets.pl
  15. +20 −0 bin/find.pl
  16. +39 −0 bin/inbox.pl
  17. +38 −0 bin/launch_task_manually.pl
  18. +77 −0 bin/mover.pl
  19. +102 −0 bin/post.pl
  20. +80 −0 bin/post_task.pl
  21. +18 −0 bin/remove.pl
  22. +20 −0 bin/report.pl
  23. +63 −0 bin/resume_crash.pl
  24. +170 −0 bin/scheduler.pl
  25. +15 −0 bin/set_password.pl
  26. +10 −0 bin/verify_twitter.pl
  27. +32 −0 bin/worker.pl
  28. +36 −0 conf/httpd.conf
  29. +66 −0 conf/perlbal-dev.conf
  30. +70 −0 conf/perlbal-prod.conf
  31. +62 −0 conf/twittary.yaml
  32. +3 −0 createdb
  33. +277 −0 data/mail-invit
  34. +23 −0 dev-local-lib/DateTime/Format/Twitter.pm
  35. +32 −0 dev-local-lib/Form/Processor/Field/EmailSize.pm
  36. +35 −0 dev-local-lib/Form/Processor/Field/Twitter.pm
  37. +18 −0 dev-local-lib/WWW/Blog/Metadata/RSD.pm
  38. +11 −0 init.pl
  39. +18 −0 lib/Catalyst/Plugin/Authentication/Store/Twittary.pm
  40. +44 −0 lib/Catalyst/Plugin/Authentication/Store/Twittary/Backend.pm
  41. +46 −0 lib/Catalyst/Plugin/Authentication/Store/Twittary/User.pm
  42. +194 −0 lib/Perlbal/Plugin/HitTracker.pm
  43. +82 −0 lib/Twittary.pm
  44. +334 −0 lib/Twittary/API/User.pm
  45. +28 −0 lib/Twittary/BlogDetector.pm
  46. +41 −0 lib/Twittary/Bootstrap.pm
  47. +46 −0 lib/Twittary/Controller/BlogDispatch.pm
  48. +17 −0 lib/Twittary/Controller/Donate.pm
  49. +289 −0 lib/Twittary/Controller/Root.pm
  50. +309 −0 lib/Twittary/Controller/SignUp.pm
  51. +171 −0 lib/Twittary/Controller/User.pm
  52. +86 −0 lib/Twittary/Core.pm
  53. +123 −0 lib/Twittary/DB.pm
  54. +41 −0 lib/Twittary/EmailSender.pm
  55. +181 −0 lib/Twittary/Fetcher.pm
  56. +21 −0 lib/Twittary/Form/SignUp/Blog.pm
  57. +21 −0 lib/Twittary/Form/User/Atom.pm
  58. +16 −0 lib/Twittary/Form/User/Email.pm
  59. +32 −0 lib/Twittary/Form/User/EmailOpt.pm
  60. +38 −0 lib/Twittary/Form/User/EmailReq.pm
  61. +26 −0 lib/Twittary/Form/User/PasswordReset.pm
  62. +21 −0 lib/Twittary/Form/User/PreferredPosting.pm
  63. +39 −0 lib/Twittary/Form/User/Prefs.pm
  64. +17 −0 lib/Twittary/Form/User/Twitter.pm
  65. +21 −0 lib/Twittary/Form/User/Xmlrpc.pm
  66. +151 −0 lib/Twittary/Formatter.pm
  67. +18 −0 lib/Twittary/Formatter/List.pm
  68. +18 −0 lib/Twittary/Formatter/Paragraph.pm
  69. +45 −0 lib/Twittary/Formatter/Title.pm
  70. +172 −0 lib/Twittary/Importer.pm
  71. +32 −0 lib/Twittary/Inbox.pm
  72. +57 −0 lib/Twittary/Inbox/Bounce.pm
  73. +54 −0 lib/Twittary/Inbox/TypePadInvites.pm
  74. +229 −0 lib/Twittary/Jobs.pm
  75. +30 −0 lib/Twittary/Model/AuthToken.pm
  76. +28 −0 lib/Twittary/Model/Hit.pm
  77. +95 −0 lib/Twittary/Model/Tweet.pm
  78. +450 −0 lib/Twittary/Model/User.pm
  79. +35 −0 lib/Twittary/Model/User/Catalyst.pm
  80. +26 −0 lib/Twittary/Poster.pm
  81. +42 −0 lib/Twittary/Poster/Atom.pm
  82. +51 −0 lib/Twittary/Poster/Email.pm
  83. +17 −0 lib/Twittary/Poster/TypePad.pm
  84. +49 −0 lib/Twittary/Poster/XMLRPC.pm
  85. +37 −0 lib/Twittary/TweetDriver.pm
  86. +137 −0 lib/Twittary/TypePad.pm
  87. +57 −0 lib/Twittary/Util.pm
  88. +53 −0 lib/Twittary/View/TT.pm
  89. +67 −0 root/about.tt
  90. +26 −0 root/atom-api-help.tt
  91. +48 −0 root/blog-compat.tt
  92. +8 −0 root/blog_form.tt
  93. +35 −0 root/blogid.tt
  94. +30 −0 root/donate/paypal.tt
  95. +11 −0 root/donate/thanks.tt
  96. +4 −0 root/dumpme.tt
  97. +8 −0 root/email-sent.tt
  98. +7 −0 root/error-invalid-user.tt
  99. +7 −0 root/error.tt
  100. BIN root/favicon.ico
  101. +15 −0 root/feedback.tt
  102. +22 −0 root/footer.tt
  103. +5 −0 root/forgotten-pass-email.tt
  104. +38 −0 root/forgotten-password.tt
  105. +14 −0 root/form/atom.tt
  106. +10 −0 root/form/email.tt
  107. +8 −0 root/form/email_account.tt
  108. +15 −0 root/form/email_auth.tt
  109. +48 −0 root/form/macros.tt
  110. +15 −0 root/form/password.tt
  111. +109 −0 root/form/prefs.tt
  112. +7 −0 root/form/twitter.tt
  113. +15 −0 root/form/wrapper.tt
  114. +13 −0 root/form/xmlrpc.tt
  115. +5 −0 root/header.tt
  116. +18 −0 root/home_loggedin_unreg.tt
  117. +22 −0 root/home_loggedout.tt
  118. +155 −0 root/home_member.tt
  119. +81 −0 root/layouts/full.tt
  120. +6 −0 root/login_box_email.tt
  121. +5 −0 root/login_box_openid.tt
  122. +10 −0 root/login_menu.tt
  123. +23 −0 root/maintenance/index.html
  124. +13 −0 root/menu.tt
  125. +6 −0 root/notfound.tt
  126. +9 −0 root/resend-email-confirmation.tt
  127. +5 −0 root/reset-password.tt
  128. +23 −0 root/sidebar.tt
  129. +22 −0 root/signin.tt
  130. +33 −0 root/signup/atom.tt
  131. +15 −0 root/signup/blog.tt
  132. +31 −0 root/signup/email.tt
  133. +27 −0 root/signup/email_auth.tt
  134. +6 −0 root/signup/prefs.tt
  135. +8 −0 root/signup/twitter.tt
  136. +38 −0 root/signup/typepad_guest.tt
  137. +23 −0 root/signup/xmlrpc.tt
  138. +368 −0 root/static/css/refresh.css
  139. BIN root/static/css/refresh/content.jpg
  140. BIN root/static/css/refresh/footer.jpg
  141. BIN root/static/css/refresh/header.jpg
  142. BIN root/static/css/refresh/headerbg.gif
  143. BIN root/static/css/refresh/menu.jpg
  144. BIN root/static/images/beta.gif
  145. BIN root/static/images/btn_120x50_built.png
  146. BIN root/static/images/btn_120x50_built_shadow.png
  147. BIN root/static/images/btn_120x50_powered.png
  148. BIN root/static/images/btn_120x50_powered_shadow.png
  149. BIN root/static/images/btn_88x31_built.png
  150. BIN root/static/images/btn_88x31_built_shadow.png
  151. BIN root/static/images/btn_88x31_powered.png
  152. BIN root/static/images/btn_88x31_powered_shadow.png
  153. BIN root/static/images/catalyst_logo.png
  154. BIN root/static/images/underconstruction.gif
  155. +105 −0 root/static/js/theonly.js
  156. +13 −0 root/static/js/tz.js
  157. BIN root/static/megaphone.jpg
  158. BIN root/static/megaphone_icon.jpg
  159. BIN root/static/megaphone_medium.jpg
  160. BIN root/static/openid.gif
  161. +45 −0 root/static/screen.css
  162. BIN root/static/sshot_loudtwitter.png
  163. BIN root/static/sshot_loudtwitter2.png
  164. BIN root/static/typepad_guest_author.png
  165. +63 −0 root/support.tt
  166. +14 −0 root/tos.tt
  167. +21 −0 root/twitter_form.tt
  168. +10 −0 root/user/atom.tt
  169. +10 −0 root/user/delete.tt
  170. +7 −0 root/user/email.tt
  171. +16 −0 root/user/prefs.tt
  172. +26 −0 root/user/test-setup.tt
  173. +8 −0 root/user/twitter.tt
  174. +7 −0 root/user/xmlrpc.tt
  175. +33 −0 root/wait-confirm-email.tt
  176. +2 −0 script/start_perlbal.sh
  177. +37 −0 script/twittary_cgi.pl
  178. +74 −0 script/twittary_create.pl
  179. +80 −0 script/twittary_fastcgi.pl
  180. +113 −0 script/twittary_server.pl
  181. +54 −0 script/twittary_test.pl
  182. +7 −0 t/01app.t
  183. +9 −0 t/02pod.t
  184. +9 −0 t/03podcoverage.t
  185. +15 −0 t/04driver.t
  186. +33 −0 t/05models.t
  187. +134 −0 t/06formatter.t
  188. +62 −0 t/07title_formatter.t
  189. +15 −0 t/lib/Fake/Tweet.pm
  190. +6 −0 t/view_TT.t
23 README
@@ -0,0 +1,23 @@
+This is Loudtwitter.com source code.
+
+Loudtwitter is a webservice that aggregates your tweets and posts them at the
+time you choose on your blog (or to any email address).
+
+STATE OF THIS CODE:
+-------------------
+
+* Note that 'master' was never pushed fully in prod, it has more features than
+ 'production' (dynamic titles, twitter name change resilience...)
+
+* Scheduling and Delivery relies on Gearman where it should rather rely
+ on TheSchwartz (with builtin retries) - and could use a mechanism scheme in general.
+
+* The project started as a hack in 2007 and grew wildely when time permitted
+ (there was no company to support it or anything)
+
+* At the present time this code is largely unmaintened
+
+WARRANTY:
+---------
+
+This code doesn't come with any warranty of any kind.
3 SETUP
@@ -0,0 +1,3 @@
+- be sure that postfix can write the log (or use .forward in ~)
+- be sure that postfix can find and excute the inbox.pl
+- be sure that the config is accessible to application users
@@ -0,0 +1,18 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use lib 'lib';
+use lib 'dev-local-lib';
+use Twittary::Model::User;
+
+my $openid_uri = shift || die "specify an url";
+my $twitter_name = shift || "loiclemeur";
+
+my $user = Twittary::Model::User->new;
+
+$user->openid_uri($openid_uri);
+$user->twitter_name($twitter_name);
+$user->post_hour('18');
+$user->post_minute(0);
+$user->save;
+print STDERR $user->user_id . "\n";
@@ -0,0 +1,37 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use lib 'lib';
+use lib 'dev-local-lib';
+use Twittary::Model::User;
+
+my $user = Twittary::Model::User->lookup(shift);
+my $n = scalar @{ $user->daily_tweets };
+
+my $auth =
+ join ", ", map { $_->value } Twittary::Model::AuthToken->search({
+ user_id => $user->user_id
+ });
+
+no warnings;
+print <<EOP;
+shid: ${ \$user->shid }
+post date: ${ \$user->post_hour }:${ \$user->post_minute } ${ \$user->timezone }
+twitter_name: ${ \$user->twitter_name }
+suspended: ${ \$user->has_suspended }
+can_post: ${ \$user->can_post } - email_verified: ${ \$user->email_verified } no fetch: ${ \$user->no_fetch }
+ppm: ${ \$user->preferred_posting_method }
+email: ${ \$user->email }
+password: ${ \$user->password }
+next_post_date: ${ \$user->next_post_date } ${ \$user->eta_before_next } local(${ \$user->next_post_date_local })
+today's tweet: $n
+auth: $auth
+last_fetched: ${ \$user->last_fetched_on_http_date }
+post failures: ${ \$user->post_failure_count }
+created_on: ${ \$user->created_on }
+endpoint_atom: ${ \$user->endpoint_atom }
+endpoint_xmlrpc: ${ \$user->endpoint_xmlrpc }
+endpoint_email: ${ \$user->endpoint_email }
+EOP
+
+
@@ -0,0 +1,24 @@
+#! /bin/sh
+
+datum=`/bin/date +%Y%m%d-%H`
+dir="/backups/sql"
+file="${dir}/backup-${datum}.sql"
+user=root
+password=none
+
+mkdir -p $dir >/dev/null 2>&1
+
+/usr/bin/mysql -p$password -e 'STOP SLAVE SQL_THREAD;'
+/usr/bin/mysqldump --user=$user -p$password twitter user auth_token > $file
+/usr/bin/mysql -p$password -e 'START SLAVE SQL_THREAD;'
+bzip2 -c $file > $file.bz2
+gpg2 --trust-model=always --yes --batch --no-tty -r kerherve -o $file.gpg -e $file.bz2
+rm $file.bz2
+rm $file
+
+## now delete old local files
+for file in "$( /usr/bin/find $dir -type f -mtime +7 )"
+do
+ /bin/rm -f ${file}.gpg
+done
+echo "done ${file}"
@@ -0,0 +1,101 @@
+#! /usr/bin/perl -w
+use strict;
+use warnings;
+use Find::Lib '../lib' => 'Twittary::Bootstrap';
+
+use Getopt::Long;
+use Time::HiRes();
+use DateTime;
+
+use Twittary::DB;
+use Twittary::Core;
+use Twittary::Importer;
+use Twittary::Core;
+use Twittary::Jobs;
+
+my %Locks;
+my %opt;
+
+GetOptions( 'no-sleep' => \$opt{no_sleep} );
+
+my $log = Twittary::Core->log;
+my $bucket_size = 5 * 60; # 5mn bucket size
+my $bucket_max = int( 24 * 3600 / $bucket_size );
+my $lock_time = $bucket_size;
+
+my ($current_bucket) = get_current_bucket($bucket_size);
+$log->debug("Starting bucket $current_bucket");
+
+## we are scheduled to post in 3h from now
+my $future_bucket = $current_bucket + 3600 * 3 / $bucket_size;
+
+my $fetchs = select_users_btw_buckets($bucket_size, $current_bucket, $future_bucket);
+
+my $fetch_count = scalar @$fetchs;
+for my $user_id (@$fetchs) {
+ Twittary::Jobs->background_task('fetch', $user_id, 'partial');
+}
+
+sub bucket_to_date {
+ my $size = shift;
+ my $bucket = shift;
+ my $bucket_0 = DateTime->now->truncate(to => 'day');
+ $bucket_0->add( seconds => $bucket * $size);
+ return Twittary::Util->dt_to_mysql($bucket_0);
+}
+
+sub get_current_bucket {
+ my $size = shift;
+ my $now = DateTime->now;
+
+ my $bucket_0 = $now->clone->truncate(to => 'day');
+ my $delta = $now->clone->subtract_datetime_absolute( $bucket_0 );
+ my $seconds = $delta->in_units('seconds');
+ my $bucket = int($seconds / $size);
+ my $remaining = $size - $seconds % $size;
+ return ($bucket, $remaining);
+}
+
+sub select_users_btw_buckets {
+ my ($bucket_size, $bucket1, $bucket2) = @_;
+
+ my $current_bucket = bucket_to_date( $bucket_size, $bucket1 );
+ my $next_bucket = bucket_to_date( $bucket_size, $bucket2 );
+
+ my $t0 = [ Time::HiRes::gettimeofday ];
+ my $now = DateTime->now;
+ my $dbh = Twittary::DB->dbh;
+
+ my $SQL = <<EOSQL;
+SELECT user_id,
+ next_post_date,
+ last_fetched_on,
+ last_post_failure_date,
+ post_failure_count
+ FROM user
+ WHERE next_post_date >= ?
+ AND next_post_date < ?
+ AND (has_suspended = 0 or has_suspended IS NULL)
+EOSQL
+
+ my $sth = $dbh->prepare_cached($SQL)
+ or die $dbh->errstr;
+ $sth->execute($current_bucket, $next_bucket);
+ $log->debug("DATES: $current_bucket, $next_bucket");
+ my $fetch = [];
+ while (my $row = $sth->fetch) {
+ my ( $user_id, $post_date_str, $fetch_date_str,
+ $fail_date_str, $count ) = @$row;
+ my $post_date = $post_date_str
+ ? Twittary::Util->mysql_to_dt($post_date_str)
+ : undef;
+ my $fetch_date = $fetch_date_str
+ ? Twittary::Util->mysql_to_dt($fetch_date_str)
+ : undef;
+
+ push @$fetch, $user_id;
+ }
+ my $runtime = Time::HiRes::tv_interval($t0);
+ $log->info(sprintf "Selected in %.2f", $runtime);
+ return $fetch;
+}
@@ -0,0 +1,9 @@
+#!/usr/bin/perl -d
+use Find::Lib '../lib' => 'Twittary::Bootstrap';
+use Twittary::API::User;
+
+sub user {
+ Twittary::API::User->find(@_);
+}
+
+1;
@@ -0,0 +1,19 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use lib 'lib';
+use lib 'dev-local-lib';
+use Twittary::Model::User;
+use Twittary::API::User;
+
+my $id = shift;
+
+my $user = Twittary::API::User->find($id)
+ or die "not found";
+
+my $ret = Twittary::API::User->delete(user => $user);
+if ($ret) {
+ print "OK\n";
+} else {
+ print "NOT OK\n";
+}
@@ -0,0 +1,9 @@
+#! /usr/bin/perl -w
+use strict;
+use warnings;
+
+my $uri = shift;
+use WWW::Blog::Metadata;
+my $meta = WWW::Blog::Metadata->extract_from_uri($uri);
+use YAML; warn Dump $meta;
+
@@ -0,0 +1,146 @@
+#! /usr/bin/perl -w
+use strict;
+use warnings;
+use Find::Lib '../lib' => 'Twittary::Bootstrap';
+
+use Getopt::Long;
+use Time::HiRes();
+use DateTime;
+
+use Gearman::Task;
+use Gearman::Client;
+
+use Twittary::DB;
+use Twittary::Core;
+use Twittary::Importer;
+use Twittary::Core;
+
+my $log = Twittary::Core->log;
+my $cfg = Twittary::Core->config;
+my $js = [ $cfg->{GearmanServers} ];
+my $client = Gearman::Client->new( job_servers => $js );
+
+my %Locks;
+my %opt;
+
+GetOptions( 'no-sleep' => \$opt{no_sleep} );
+
+my $bucket_size = 60 * 60; # every hour
+my $bucket_max = int( 24 * 3600 / $bucket_size );
+
+my ($bucket) = get_current_bucket($bucket_size);
+while (1) {
+ $log->debug("Starting bucket $bucket");
+ my $t0 = [ Time::HiRes::gettimeofday ];
+
+ ## we do two selects in far less time than a bucket duration (in theory)
+ my $posts = select_users_for_bucket($bucket_size, $bucket);
+ for my $user_id (@{ $posts }) {
+ task('post', $user_id);
+ }
+
+ my $runtime = Time::HiRes::tv_interval($t0);
+ my ($target, $remaining) = get_current_bucket($bucket_size);
+ if ($target > $bucket) {
+ $log->info(sprintf "Done in %.2f, posts: %d, switched bucket %d vs. %d",
+ $runtime, scalar @$posts, $target, $bucket);
+ }
+ else {
+ $log->info(sprintf "Done in %.2f, posts: %d, sleeping for %ds",
+ $runtime, scalar @$posts, $remaining);
+ sleep $remaining;
+ }
+
+ ## next bucket
+ $bucket = ++$bucket % $bucket_max;
+}
+
+sub bucket_to_date {
+ my $size = shift;
+ my $bucket = shift;
+ my $bucket_0 = DateTime->now->truncate(to => 'day');
+ $bucket_0->add( seconds => $bucket * $size);
+ return Twittary::Util->dt_to_mysql($bucket_0);
+}
+
+sub get_current_bucket {
+ my $size = shift;
+ my $now = DateTime->now;
+
+ my $bucket_0 = $now->clone->truncate(to => 'day');
+ my $delta = $now->clone->subtract_datetime_absolute( $bucket_0 );
+ my $seconds = $delta->in_units('seconds');
+ $log->debug("B0 $bucket_0 - $seconds");
+ my $bucket = int($seconds / $size);
+ my $remaining = $size - ($seconds % $size);
+ return ($bucket, $remaining);
+}
+
+sub select_users_for_bucket {
+ my ($bucket_size, $bucket) = @_;
+
+ # rewind 24 buckets
+ my $begin_bucket = bucket_to_date( $bucket_size, $bucket - 25);
+ my $end_bucket = bucket_to_date( $bucket_size, $bucket - 1);
+
+ my $t0 = [ Time::HiRes::gettimeofday ];
+ my $now = DateTime->now;
+ my $dbh = Twittary::DB->dbh;
+
+ my $SQL = <<EOSQL;
+SELECT user_id,
+ next_post_date,
+ last_fetched_on,
+ last_post_failure_date,
+ post_failure_count
+ FROM user
+ WHERE next_post_date >= ?
+ AND next_post_date < ?
+ AND (has_suspended = 0 or has_suspended IS NULL)
+EOSQL
+
+ my $sth = $dbh->prepare_cached($SQL)
+ or die $dbh->errstr;
+ $sth->execute($begin_bucket, $end_bucket);
+ $log->debug("DATES: $begin_bucket, $end_bucket");
+ my $post = [];
+ while (my $row = $sth->fetch) {
+ my ( $user_id, $post_date_str, $fetch_date_str,
+ $fail_date_str, $count ) = @$row;
+ my $post_date = $post_date_str
+ ? Twittary::Util->mysql_to_dt($post_date_str)
+ : undef;
+ my $fetch_date = $fetch_date_str
+ ? Twittary::Util->mysql_to_dt($fetch_date_str)
+ : undef;
+
+ ## now determine user_id to post, exclude failing users
+ if ($fail_date_str) {
+ # XXX reset this fields on successful posting
+ my $fail_date = Twittary::Util->mysql_to_dt($fail_date_str);
+ my $quarantine = DateTime::Duration->new( seconds => 2 ** $count );
+ next unless $fail_date->clone->add_duration($quarantine) < $now;
+ }
+ push @$post, $user_id;
+ }
+ my $runtime = Time::HiRes::tv_interval($t0);
+ $log->info(sprintf "Selected in %.2f", $runtime);
+ return $post;
+}
+
+sub task {
+ my $type = shift;
+ my $user_id = shift;
+ my $task = Gearman::Task->new(
+ $type,
+ \Storable::nfreeze({
+ user_id => $user_id,
+ }),
+ {
+ ## XXX i don't trust uniq implementation. Test
+ #uniq => '-',
+ uniq => "$type-$user_id",
+ },
+ );
+ $client->dispatch_background($task);
+}
Oops, something went wrong.

0 comments on commit a2107cd

Please sign in to comment.