Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Make stuff "work"

  • Loading branch information...
commit 89209e433a41d3a6d1cdaea2bc68b1cc0c1a1756 1 parent 48de3b9
@holdenk authored
View
1  Makefile.PL
@@ -17,6 +17,7 @@ WriteMakefile
'Text::SpellChecker' => 0,
'Pithub' => 0,
'YAML::Any' => 0,
+ 'Slurp' => 0,
},
'linkext' => { LINKTYPE => '' }, # no link needed
'dist' =>
View
2  Unicorn/Cpp.pm
@@ -21,7 +21,7 @@ sub fix_cpp {
if (-f $spatchdir."/".$spatch_file && $spatch_file =~ /\.cocci$/) {
print "running spatch $spatchdir/$spatch_file $file\n";
print "got:\n";
- print `$spatchexec --in-place --sp-file $spatchdir/$spatch_file $file`;
+ print `$spatchexec -c++ --in-place --sp-file $spatchdir/$spatch_file $file`;
}
}
closedir($spatch_files);
View
2  Unicorn/Errorcheck.pm
@@ -15,7 +15,7 @@ sub fix_go {
return 1;
}
sub check_cpp {
- return 0;
+ return 1;
}
sub fix_cpp {
my $file = shift @_;
View
3  Unicorn/Wordlist.pm
@@ -545,7 +545,7 @@ sub fix_text {
my $text = shift @_;
foreach my $k (keys %common) {
if ($text =~ s/\b($k)\b/preserve_case($1,$common{$k})/egi) {
- print "Swapped on $k update to $common{$k}\n";
+ 1;
}
}
return $text;
@@ -569,7 +569,6 @@ sub check_common {
foreach my $w ( @words ) {
my $lcw = lc( $w );
if ( exists $common{ $lcw } ) {
- print "Error was $lcw\n";
return 1;
}
}
View
19 bigquerytargets.pl
@@ -1,3 +1,20 @@
# Gets top repos from gitarchive.org
# Please run a bigtable query beforehand to make sure you have bq credentials setup
-`bq --project_id holdensmagicalunicorn query
+use strict;
+use warnings;
+use Slurp;
+
+my $query = slurp("bigtablequery.bq");
+my @languages = ("C++","C","OCAML","Ruby","Perl","PHP");
+for my $language (@languages) {
+ my $newquery = $query;
+ $newquery =~ s/LANG/$language/g;
+ $newquery =~ s/MYLIMIT/15000/g;
+ print "running $newquery\n";
+ my $hoboin;
+ open($hoboin, "bq --project_id holdensmagicalunicorn --format csv query \"$newquery\"|");
+ while (my $line = <$hoboin>) {
+ my @murh = split(/\,/, $line);
+ print $murh[0]."\n";
+ }
+}
View
10 bigtablequery.bq
@@ -1,8 +1,8 @@
-/* top 15000 repos for C++ by number of pushes */
-SELECT repository_name, count(repository_name) as pushes, repository_description, repository_url
+/* top 15000 repos for LANG by number of pushes */
+SELECT repository_url, repository_name, count(repository_name) as pushes, repository_description
FROM [githubarchive:github.timeline]
-WHERE type="PushEvent"
-AND (repository_language="C++")
+WHERE type=\"PushEvent\"
+AND (repository_language=\"LANG\")
GROUP BY repository_name, repository_description, repository_url
ORDER BY pushes DESC
-LIMIT 15000;
+LIMIT MYLIMIT;
View
89 fix_pandas.pl
@@ -11,6 +11,8 @@
use Unicorn::Blacklist qw{ ok_to_update };
use Unicorn::Settings qw{ settings }:
+require "shared_fix.pl";
+
my $p = Pithub->new;
my $c = 0;
@@ -46,13 +48,9 @@
sub handle_url {
my $url = shift @_;
print "looking at $url\n";
- if ($url =~ /http.*\/(.*?)\/(.*?)\/(raw\/|)(master|development|\w+)\/(.*)/) {
+ if ($url =~ /http.*\/.*github\.com\/(.*?)\/(.*?)(\/|$).*/) {
my $ruser = $1;
my $repo = $2;
- my $file = $4;
- print "u:".$ruser."\n";
- print "r:".$repo."\n";
- print "f:".$file."\n";
my $result = $p->repos->get( user => $ruser , repo => $repo);
my $traverse = 0;
#Do we need to go up a level?
@@ -133,84 +131,3 @@ sub generate_twitter_msg {
}
return $message;
}
-sub handle_files {
- print "handle_files called\n";
- my @files = @_;
- my @handlers = (handle_group("Fixing typos in README",qr/\/README(\.txt|\.rtf|\.md|\.pm|\.m\w+)$/,\&check_common,\&fix_text),
- handle_group("Fixing old PHP calls",qr/\.php$/,\&check_php,\&fix_php),
- handle_group("Updating shell scripts",qr/\/\w(\.sh|\.bash|)$/,\&check_shell,\&fix_shell),
- handle_group("Fixing deprecated django",qr/\.py$/,\&check_py,\&fix_py),
- handle_group_cmd("Fixing go formatting",qr/\.go$/,\&check_go,\&fix_go));
- my @handler_names = ("typos","deprecated php","portable shell","deprecated django","go fix");
- print "have ".$#files." and ".$#handlers." to use\n";
- my $i = 0;
- my $short_msg = "Fix ";
- my @changes = ();
- while ($i < $#handlers+1) {
- print "running $i\n";
- print "Running handler $i / ".$handler_names[$i]."\n";
- my $r = $handlers[$i](@files);
- if ($r) {
- push @changes, $handler_names[$i];
- }
- $i++;
- }
- return @changes;
-}
-sub handle_group {
- my $git_message = shift @_;
- my $gate_regex = shift @_;
- my $gate_function = shift @_;
- my $fix_function = shift @_;
- return sub {
- my $changes = 0;
- my @files = @_;
- foreach my $file (@files) {
- if ($file !~ /\/\.git\// && $file =~ $gate_regex) {
- open (my $in, "<", "$file") or die "Unable to open $file";
- my $t = do { local $/ = <$in> };
- close($in);
- #Is there a spelling mistake?
- if ($gate_function->($file, $t)) {
- open (my $out, ">", "$file") or die "Unable to open $file";
- print $out $fix_function->($file, $t);
- close ($out);
- }
- }
- }
- #Determine if we have made any difference
- `cd foo/*;git diff --exit-code`;
- if ($? != 0) {
- #Yup
- `cd foo/*;git commit -a -m \"$git_message\";git push; sleep 1; git push`;
- return 1;
- }
- #Nope no changes
- return 0;
- }
-}
-
-sub handle_group_cmd {
- my $git_message = shift @_;
- my $gate_regex = shift @_;
- my $gate_function = shift @_;
- my $fix_function = shift @_;
- return sub {
- my $changes = 0;
- my @files = @_;
- foreach my $file (@files) {
- if ($file !~ /\/\.git\// && $file =~ $gate_regex) {
- $fix_function->($file);
- }
- }
- #Determine if we have made any difference
- `cd foo/*;git diff --exit-code`;
- if ($? != 0) {
- #Yup
- `cd foo/*;git commit -a -m \"$git_message\";git push; sleep 1; git push`;
- return 1;
- }
- #Nope no changes
- return 0;
- }
-}
View
90 main.pl
@@ -2,6 +2,90 @@
# The purpose of this is to turn the entire pipeline, possibly on multiple machines
# First step is to get the list of projects to consider which runs on a single host
# but we don't have to block on it
-open (BINGTARGETS , "perl targets.pl| tee bingtargets |");
-open (GHTARGETS, "perl targets2.pl| tee ghtargets |");
-open (BQTARGETS, "perl bigquerytargets.pl| tee bqtargets|");
+use strict;
+use warnings;
+use IO::Select;
+use IPC::Open2;
+
+my $remoteoutselect = IO::Select->new();
+my $remoteinselect = IO::Select->new();
+my $badrepos;
+
+sub main() {
+ setup_output();
+ my ($bingin,$ghin,$bqin);
+
+ open ($bingin , "perl targets.pl\| tee bingtargets |");
+ open ($ghin, "perl targets2.pl\| tee ghtargets |");
+ open ($bqin, "perl bigquerytargets.pl\| tee bqtargets|");
+ open ($fixstuff, "|perl fix_pandas.pl\| tee fixdata");
+ my $s = IO::Select->new();
+ $s->add($bingin);
+ $s->add($ghin);
+ $s->add($bqin);
+ while (my @ready = $s->can_read) {
+ foreach my $fh (@ready) {
+ my $line = <$fh>;
+ handle_line($line);
+ }
+ }
+ # Read the input back from the hosts as it becomes available
+ while (my @ready = $remoteinselect->can_read(60)) {
+ if (@ready == ()) {
+ break;
+ }
+ for my $fh (@ready) {
+ handle_possible_repo(<$fh>);
+ }
+ }
+ # Tell all of the children we are done
+ @outputhandles = $remoteoutselect->can_write;
+ for my $fh (@outputhandles) {
+ print $fh "quitquitquit\n";
+ sleep 5;
+ print $fh "exit\n";
+ }
+ # Read any remaining input back from the hosts
+ while (my @ready = $remoteinselect->can_read(60)) {
+ if (@ready == ()) {
+ break;
+ }
+ for my $fh (@ready) {
+ handle_possible_repo(<$fh>);
+ }
+ }
+ close ($badrepos);
+}
+
+sub handle_possible_repo {
+ my $repo = shift @_;
+ if ($repo =~ /http/) {
+ print $badrepos $repo;
+ print $fixstuff $repo;
+ }
+}
+
+sub setup_output {
+ my $hosts;
+ open ($hosts, "hosts.txt");
+ while (my $hostline = <$hosts>) {
+ @murh = split(/\:/,$hostline);
+ my $hostname = $murh[0];
+ my $pwd = $murh[1];
+ my ($child_out,$child_in);
+ open2($child_out, $child_in, "ssh $hostname");
+ #hack
+ sleep 10;
+ print $child_out "mkdir $pwd;cd $pwd;cp ~/mymagic.tar ./;tar -xvf mymagic.tar;perl verify.pl";
+ $remoteoutselect->add($child_out);
+ $remoteinselect->add($child_in);
+ }
+ open ($badrepos , ">badrepos.txt");
+}
+
+sub handle_line {
+ my $line = shift @_;
+
+}
+
+main();
View
71 verify.pl
@@ -0,0 +1,71 @@
+#!/usr/bin/perl
+
+#The purpose of this code is to see if we might make a change to something otherwise output none
+use strict;
+use warnings;
+$| = 1;
+use LWP::UserAgent;
+use Net::Twitter;
+use Pithub;
+use Data::Dumper;
+
+use strict;
+use Unicorn::Wordlist qw{fix_text check_common};
+use Unicorn::Errorcheck qw{check_php fix_php check_py fix_py check_go fix_go check_cpp fix_cpp check_shell fix_shell};
+use Unicorn::Blacklist qw{ ok_to_update };
+#use Unicorn::Settings qw{ settings }:
+
+require "shared_fix.pl";
+
+my $p = Pithub->new;
+while (my $l = <>) {
+ if ($l =~ /github\.com\/(.*?)\s*$/) {
+ my $url = "https://www.github.com/".$1;
+ $url =~ s/raw\/.*?\//raw\/master\//;
+ handle_url($url);
+ } elsif ($l =~ /quitquit/) {
+ exit;
+ } else {
+ print "fuck\n";
+ }
+}
+
+sub handle_url {
+ my $url = shift @_;
+ if ($url =~ /http.*\/.*github\.com\/(.*?)\/(.*?)(\/|$).*/) {
+ my $ruser = $1;
+ my $repo = $2;
+ my $result = $p->repos->get( user => $ruser , repo => $repo);
+ my $traverse = 0;
+ #Do we need to go up a level?
+ while ($traverse < 10 && $result->content->{source}) {
+ my $above = $result->content->{source}->{url};
+ if ($above =~ /repos\/(.*?)\/(.*)$/) {
+ $ruser = $1;
+ $repo = $2;
+ }
+ $result = $p->repos->get( user => $ruser , repo => $repo);
+ $traverse++;
+ }
+ # Try and get the repo
+ my $clone_url = $result->content->{clone_url};
+ `rm -rf foo && mkdir -p foo && cd foo && git clone "$clone_url" && cd *`;
+ #Get the files
+ my @all_files;
+ open (my $files,"find ./foo/|");
+ while (my $file = <$files>) {
+ chomp ($file);
+ push @all_files, $file;
+ }
+ close ($files);
+ #Now we iterate through each of the processors so the git commit messages are grouped logically
+ my @changes = handle_files(@all_files);
+ #Did we change anything?
+ if ($#changes > 0) {
+ print "$url\n";
+ return 1;
+ } else {
+ print "no win\n";
+ }
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.