diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d02c524 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,32 @@ +Copyright (c) 2011, Yahoo! Inc. All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the following +conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of Yahoo! Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of Yahoo! Inc. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/Seco_Range/Range.pm b/Seco_Range/Range.pm new file mode 100644 index 0000000..d2bf629 --- /dev/null +++ b/Seco_Range/Range.pm @@ -0,0 +1,729 @@ +=head1 NAME + +Range - deal with Seco ranges. + +=head1 SYNOPSIS + +use Range qw/:common/; + +my @nodes = expand_range('%ks301'); + +my $nodes = compress_range(\@nodes); + +my $same_nodes = compress_range(@nodes); + +my @sorted_nodes = sorted_expand_range('@ALL'); + +=head1 DESCRIPTION + +Do stuff with ranges. + +Expand ranges and cluster definitions. +Compress (a ref to) an array of nodes into a compact string rep. + +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +=cut + +package Seco::Range; + +use warnings; +use strict; +use Exporter; +use IO::File; +use Sys::Hostname; +use Storable; +use Carp; + +use constant NODE_CLUSTER => '/home/seco/candy/whoismycluster/node_cluster.dat'; + +use vars qw(@ISA @EXPORT_OK @EXPORT %EXPORT_TAGS $VERSION $recursion); +use vars qw($IGOR); +use Log::Log4perl qw(:easy); + +@ISA = qw/Exporter/; +@EXPORT_OK = qw/expand_range get_cluster_nodes range_set_altpath + compress_range sorted_expand_range nodes_parser get_expanded_clusters/; +%EXPORT_TAGS = ( + all => [@EXPORT_OK], + common => [qw/expand_range compress_range sorted_expand_range nodes_parser + get_expanded_clusters/]); +@EXPORT = (); # don't pollute the namespace - use Range qw/:common/ + +$VERSION = "1.4.1"; +$recursion = 0; + +#eval <<'MEMOIZE'; +#use Memoize; # hmmm caching +#memoize('expand_range'); +#memoize('_get_cluster_keys'); +#MEMOIZE + +my $node_regex = qr/ + ([-\w.]*?) # prefix + (\d+) # followed by the start of the range + (\.[-A-Za-z\d.]*[-A-Za-z]+[-A-Za-z\d.]*)? # optional domain +/x; + +my ($balanced_parens, $balanced_braces); +if ($] > 5.006) { + $balanced_parens = qr/ + \( + (?: (?> [^( )]+ ) # no backtracking here (normal chars) + | # or + (??{ $balanced_parens }) )* # recursive matching here + \) + /x; + + $balanced_braces = qr/ + { + (?: (?> [^{ }]+ ) # no backtracking here (normal chars) + | + (??{ $balanced_braces }) )* # recursive matching here + } + /x; +} else { + # F*ing solaris machines + $balanced_parens = qr/ + \( + (?: (?> [^()]+ ) | \( (?> [^()]+) \)+)* + \) + /x; + $balanced_braces = qr/ + { + (?: (?> [^{}]+ ) | { (?> [^{}]+) }+)* + } + /x; +} + +my $range_re = qr@ + (?:^|,) # beginning of the string or , is start of range + ( # start capturing our range + [^(){}/,]* # normal characters (not {} or / or ()) + (?: + $balanced_parens + | + (?:-|&) \/ [^\/]+ \/ # a reg.ex. + | # or + (?:-|&) \| [^\|]+ \| + | + $balanced_braces + [^{},]* # followed by optional normal chars + )+ # one or more times + | # or it can be a simple thing (no braces involved) + (?> # never backtrack over these + [^(){},/]+) # normal chars + )@x; + +my $range_altpath; +sub range_set_altpath { + my $new_path = shift; + return $range_altpath = $new_path; +} + +sub _sort_nodes { + my $ref_nodes = shift; + + my @sorted = + map { $_->[0] } + sort { $a->[1] cmp $b->[1] || + $a->[3] cmp $b->[3] || + $a->[2] <=> $b->[2] || + $a->[0] cmp $b->[0] } + map { if (/^$node_regex$/) { + [$_, defined $1 ? $1 : "", defined $2 ? $2 : 0, + defined $3 ? $3 : ""] + } else { [$_, "", 0, ""] } + } @$ref_nodes; + return @sorted; +} + +# compress_range related stuff +sub _get_group { + my ($prefix, $digits, $count, $suffix) = @_; + + $prefix = "" unless defined $prefix; + my $len_digits = length($digits); + my $node_fmt = "\%s\%0${len_digits}d"; + my $group_fmt = "$node_fmt-" . substr($node_fmt, 2); + my $group = sprintf($group_fmt, $prefix, $digits, + $digits + $count); + + $suffix = "" unless defined $suffix; + return $group . $suffix; +} + +sub compress_range { + my @nodes = @_; + if (@nodes == 1 and ref $nodes[0]) { + @nodes = @{$nodes[0]}; + } + + unless (@nodes) { + _range_warn("No nodes specified."); + return; + } + + @nodes = _sort_nodes(\@nodes); + + my @result; + my ($prev_prefix, $prev_digits, $prev_suffix) = ("", undef, ""); + my $prev_n; + my ($prefix, $digits, $suffix); + my $count = 0; + + for my $n (@nodes) { + if ($n =~ /^$node_regex$/) { + ($prefix, $digits, $suffix) = ($1, $2, $3); + $prefix = "" unless defined $prefix; + $suffix = "" unless defined $suffix; + #print defined $prefix ? $prefix : "(undef)", " - ", defined $digits ? $digits : "(undef)", " - ", defined $suffix ? $suffix : "(undef)", "\n"; + } else { + ($prefix, $digits, $suffix) = ($n, undef, undef); + } + if (defined $digits and + $prefix eq $prev_prefix and + $suffix eq $prev_suffix and + defined $prev_digits and + $digits == $prev_digits + $count + 1) { + $count++; + next; + } + if (defined $prev_n) { + if ($count > 0) { + push @result, _get_group($prev_prefix, $prev_digits, $count, + $prev_suffix); + } else { + push @result, $prev_n; + } + } + + $prev_n = $n; + $prev_prefix = $prefix; + $prev_digits = $digits; + $prev_suffix = $suffix; + $count = 0; + } + + if ($count > 0) { + push @result, _get_group($prev_prefix, $prev_digits, $count, + $prev_suffix); + } else { + push @result, $prev_n; + } + + return join(",", @result); +} + +sub nodes_parser { + my ($c, $r, $x) = @_; + my @range = (); + if (defined $r) { + push @range, $r; + } + if (defined $c) { push @range, '%' . $c; } + if (defined $x) { push @range, "-($x)" }; + + return sorted_expand_range(join(",", @range)); +} + +sub sorted_expand_range { + my @nodes = expand_range(@_); + return _sort_nodes(\@nodes); +} + +my %current_clusters; +sub get_expanded_clusters { + return [keys %current_clusters]; +} + +sub expand_range { + my ($ranges) = @_; return () unless defined $ranges; + + local $recursion = $recursion + 1; + confess "Max recursion hit in expand_range" + if $recursion > 20; + + DEBUG("expand_range($ranges)"); + if ($recursion == 1) { + DEBUG("resetting current_clusters"); + %current_clusters = () ; + } + + + my %nodes; + for ($ranges) { + s/\s+//g; # Get rid of spaces + s/^"(.*)"$/$1/; # Dequote if quoted + + # Igor has no notion of expanding "hosts" vs "groups" so + # we're only going to use a single syntax (%HOST:) + #s/@([A-Za-z0-9][^,]+)/%GROUPS:$1/g; # shorthand notation for groups + s/@([A-Za-z0-9][^,]+)/%HOSTS:$1/g; # shorthand notation for hosts + } + + my @ranges = $ranges =~ m/$range_re/g; + for my $range (@ranges) { + # see if it's a special range + if (substr($range,0,1) eq "(") { + my $parens_text = $range; + my $content = ($parens_text =~ m/^($balanced_parens)$/)[0]; + $content = substr($content, 1, -1); + my @nodes = expand_range($content); + @nodes{@nodes} = undef; + } elsif ($range =~ m{^&\|([^|]+)\|} || + $range =~ m{^&\/([^\/]+)\/}) { + # filter + my $regex = qr/$1/; + while (my ($k, $v) = each(%nodes)) { + delete $nodes{$k} unless $k =~ /$regex/; + } + } elsif ($range =~ m{^-\|([^|]+)\|} || + $range =~ m{^-\/([^\/]+)\/}) { + # filter not + my $regex = qr/$1/; + while (my ($k, $v) = each(%nodes)) { + delete $nodes{$k} if $k =~ /$regex/; + } + } elsif ($range =~ /^(%|-|&|\^|\*)(.+)$/) { + # special operators that modify the rest of the range + my ($special_op, $rest) = ($1, $2); + if ($special_op eq "%") { + # it's a cluster + my @nodes = get_cluster_nodes($rest); + @nodes{@nodes} = undef; + } elsif ($special_op eq "-") { + # delete nodes from nodes + my @nodes = expand_range($rest); + delete @nodes{@nodes}; + } elsif ($special_op eq "&") { + # intersection + my @common_nodes = (); + my @nodes = expand_range($rest); + for my $node (@nodes) { + push @common_nodes, $node if exists $nodes{$node}; + } + %nodes = (); @nodes{@common_nodes} = undef; + } elsif ($special_op eq "^") { + my @nodes = expand_range($rest); + my @admins = _get_admins_for(@nodes); + @nodes{@admins} = undef; + } elsif ($special_op eq "*") { + my @nodes = expand_range($rest); + my @clusters = _get_clusters_for(@nodes); + @nodes{@clusters} = undef; + } + } else { + # simple range + if ($range =~ /^ + $node_regex + - # our separator is '-' + \1? # the prefix again, which is optional + (\d+) # and the end of the range + ((?(3) \3 | # if the domain matched before, we want it here + (?:\.[-A-Za-z\d.]+)?)) # if it didn't then we can have a new + # one here, like foo1-3.search + $ + /x) + { + my ($prefix, $start, $suf1, $end, $suf2) = ($1, $2, $3, $4, $5); + $prefix = "" unless defined $prefix; + my $suffix = ""; + if (defined $suf1 and defined $suf2) { + if ($suf1 ne $suf2) { + warn "Different suffixes: $suf1 $suf2"; + } + $suffix = $suf1; + } elsif (defined $suf2) { + $suffix = $suf2; + } + my $len = length($start); + # pad $end with leading characters from start so we can + # type 01-3 and expand that to 01,02,03 or maybe + # ks301000-9 for ks301000-301009 + my $len_end = length($end); + $end = substr($start, 0, $len - $len_end) . $end + if $len_end < $len; + my @nodes = map {$_ = sprintf("$prefix%0${len}d$suffix", $_) } + ($start .. $end); + @nodes{@nodes} = undef; + } elsif ($range =~ /{/) { # } + my @nodes = expand_range(join(",", _expand_braces($range))); + @nodes{@nodes} = undef; + } else { + # single machine + $nodes{$range} = undef; + } + } + } + return keys %nodes; +} + +sub get_cluster_nodes { + my $cluster = shift; + $cluster =~ s/HOSTS://; + $current_clusters{$cluster} = 1; + DEBUG("expanding $cluster"); + + my $role = $IGOR->get_role(split(/\./,$cluster,2)); + if ($IGOR->{'use_expand_cache'}) { + my $members = $role->get_members(); + my $hosts = $members->expand(); + my $base = $members->based_on(); + for (@$base) { + $current_clusters{$_} = 1; + } + return @$hosts; + } + my $members = $role->get_members()->get_range(); + DEBUG(" $members"); + my $nodes = [expand_range($members)]; + + return @$nodes; + +=for ignoring + + my $cluster = shift; + my @clusters = expand_range($cluster); + my @result; + + for $cluster (@clusters) { + die "Malformed cluster name $cluster" + unless $cluster =~ /^([-\w.]+):?([-\w.]+)?$/; + $cluster = $1; + my $part = defined($2) ? $2 : "CLUSTER"; + my %keys = _get_cluster_keys($cluster); + push @result, expand_range($keys{$part}); + } + return @result; + +=cut + +} + +{ + my %nodes_admin; + + sub _populate_nodes_admin { + my @admins = expand_range('%HOSTS:KEYS'); + for my $admin (@admins) { + my @admin_nodes = expand_range("\%HOSTS:$admin"); + for my $node (@admin_nodes) { + $nodes_admin{$node} = $admin; + } + } + } + + sub _get_admins_for { + my @nodes = @_; + _populate_nodes_admin() unless %nodes_admin; + my %results; + for my $node (@nodes) { + my $admin = $nodes_admin{$node}; + if ($admin) { + $results{$admin}++; + } else { + _range_warn("$node: admin not found"); + } + } + return keys %results; + } +} + +{ + my $nodes_cluster; + sub _populate_nodes_cluster { + $nodes_cluster = retrieve(NODE_CLUSTER); + unless ($nodes_cluster) { + _range_warn("Can't read node_cluster.dat"); + return; + } + } + + sub _get_clusters_for { + my @nodes = @_; + _populate_nodes_cluster() unless $nodes_cluster; + my %results; + for my $node (@nodes) { + my $cluster = $nodes_cluster->{$node}; + if ($cluster) { + $results{$cluster}++; + } else { + _range_warn("$node: cluster not found"); + } + } + return keys %results; + } +} + + +sub _expand_braces { + my $range = shift; + my @todo = ($range); + my @results; + local $_; + + while (@todo) { + $_ = shift(@todo); + if (/^(.*)($balanced_braces)(.*)$/) { + my ($pre, $braces, $post) = ($1, $2, $3); + my @braces = expand_range(substr($braces, 1, -1)); + for my $elt (@braces) { + if ($pre =~ /{/) { + push @todo, "$pre$elt$post"; + } else { + push @results, "$pre$elt$post"; + } + } + } else { + push @results, $_; + } + } + return @results; +} + +sub _get_cluster_keys { + my $cluster = shift; + my @lines = _read_cluster_file($cluster); + return unless @lines; + + my (%keys, $current_key, @current_range); + + for (@lines) { + s/#.*$//; s/\s+$//; + s/\$(\w+)/\%$cluster:$1/g; # Turn $MACRO into %cluster:MACRO + next unless /\S/; + + my $joinsep; + if (/^\s/ && $current_key) { + if (/^\s+INCLUDE/) { + s/^\s+INCLUDE\s+//; + } elsif (/^\s+EXCLUDE/) { + s/^\s+EXCLUDE\s+(.*)/-($1)/; + } else { + die "RangeError: $_: don't know how to parse that"; + } + + s/\s+//; # strip white space + push @current_range, $_; + } else { # New Key + # save old key info if it exists + $keys{$current_key} = join(",", @current_range) if $current_key; + + $current_key = $_; + @current_range = (); + } + } + + $keys{$current_key} = join(",", @current_range) if $current_key; + $keys{KEYS} = join(",", keys %keys); + $keys{UP} = "\%${cluster}"; + $keys{DOWN} = "\%${cluster}:ALL,-\%${cluster}:CLUSTER"; + $keys{VIPS} = _get_cluster_vips($cluster); + return %keys; +} + +sub _get_cluster_file { + my $cluster = shift; + my ($fh, $filename); + my $alt = $range_altpath || ""; + + ($filename) = grep( -e $_ , + "$alt/$cluster/tools/conf/nodes.cf", + "$alt/$cluster/nodes.cf", + "/home/seco/tools/conf/$cluster/nodes.cf", + "/usr/local/gemclient/$cluster/nodes.cf"); + $filename or _range_warn("$cluster: missing on this machine"); + return $filename; +} + +sub _read_big_file { + my $filename = shift; + open my $big_fh, '<', $filename or do { + _range_warn("$filename: $!"); + return; + }; + + my @result; + while (<$big_fh>) { + if (/^\$INCLUDE\s+"([^"]+)"/) { + my $include = $1; + my $relative_dir = "./"; + if ($include !~ m{^/}) { + # it's a relative PATH, prepend the dir for the cur file + if ($filename =~ m{^(.*/)}) { + $relative_dir = $1; + } + } + + push @result, _read_big_file("$relative_dir$filename"); + } else { + push @result, $_; + } + } + close $big_fh; + return wantarray() ? @result : join("", @result); +} + +sub _read_cluster_file { + my $cluster = shift; + my $filename = _get_cluster_file($cluster); + my @lines = _read_big_file($filename); + + # TODO: parse $INCLUDE + return @lines; +} + +sub _range_warn { + my $warn = shift; + warn "$warn\n" if -t STDIN && -t STDOUT; +} + +sub _open_cluster_vips { + my $cluster = shift; + my ($fh, $filename); + + my $alt = $range_altpath || ""; + ($filename) = grep ( -e $_, + "$alt/$cluster/tools/conf/vips.cf", + "$alt/$cluster/vips.cf", + "/home/seco/tools/conf/$cluster/vips.cf"); + $filename ||= "/dev/null"; + $fh = new IO::File "$filename", "r"; + return $fh; +} + +sub _get_cluster_vips { + my $cluster = shift; + my $fh = _open_cluster_vips($cluster); + my (@vips); + + while (<$fh>) { + s/#.*$//; + s/\s+$//; + next unless /\S/; + my($vip) = split(/\s+/); + push(@vips,$vip) if ($vip =~ m/./); + } + close($fh); + return join(",",@vips); +} + +1; + +__END__ + +=head1 FUNCTIONS + +=head2 compress_range + + $string = compress_range(\@nodes) + + $string = compress_range(@nodes) + +=head2 expand_range + + @nodes = expand_range($range) # @nodes in random order, faster + +=head2 sorted_expand_range + + @nodes = sorted_expand_range($range) # @nodes in a nice order + +=head2 get_cluster_nodes + + @nodes = get_cluster_nodes("ks301"); # random order + +=head2 range_set_altpath + + range_set_altpath("/usr/local/gemclient") + # look for cluster definitions in the directory specified + +=head1 RANGE SYNTAX + +=head2 SIMPLE RANGES + + node1,node2,node3,node4 == node1-node4 == node1-4 + + node1000-1099 == node1000-99 # auto pads digits to the end of the range + + 1-100 # numeric only ranges + + foo1-2.search.scd.yahoo.com == + foo1.search.scd.yahoo.com-foo2.search.scd.yahoo.com # domain support + + 209.131.40.1-209.131.40.255 == 209.131.40.1-255 # IP ranges + +=head2 CLUSTERS + + %ks301 == nodes defined in ks301/nodes.cf - Default Section CLUSTER + + %ks301:ALL == nodes defined in a specific section of ks301/nodes.cf + + %ks301:VIPS == IPs in ks301/vips.cf + +=head2 SPECIAL CLUSTERS + + %HOSTS and %GROUPS + + %HOSTS:haides has all the hosts that haides is responsible for. + + @haides is a shortcut for the above. + + %GROUPS:ADMIN has all the machines in the group ADMIN + + @ADMIN is a shortcut. + + +=head2 OPERATIONS + + range1,range2 == union + + range1,-range2 == set difference + + range1,&range2 == intersection + + ^range1 == admins for the nodes in range1 + + range1,-(range2,range3) == () can be used for grouping + + range1,&|regex| # all nodes in range1 that match regex + + range1,-|regex| # all nodes in range1 that do not match regex + + /regex/ == all nodes that match regex (it does matching against @ALL) + + The difference between |regex| and /regex/ is that |regex| does the + matching against the left side of the expression, while /regex/ does + the matching against all nodes. Therefore + + fornode.pl -r /ks30/ -l # makes sense + + fornode.pl -r |ks30| -l # doesn't make sense since there's nothing to the left + + +=head2 MORE ADVANCED RANGES + + foo{1,3,5} == foo1,foo3,foo5 + + %ks30{1,3} == %ks301,%ks303 + + %ks301-7 == nodes in clusters ks301 to ks307 + + %all:KEYS == all defined sections in cluster all + + %{%all} == expands all clusters in %all + + %all:sc5,-({f,k}s301-7) == names for clusters in sc5 except ks301-7,fs301-7 + + %all:sc5,-|ks| == clusters in sc5, except those matching ks + +=head1 BUGS + +Need more docs + +=head1 AUTHOR + +Daniel Muino + +=cut diff --git a/libcrange/index.yaml b/libcrange/index.yaml new file mode 100644 index 0000000..acc9f5f --- /dev/null +++ b/libcrange/index.yaml @@ -0,0 +1,7 @@ +default: + name: libcrange + summary: C version of librange + version: '1.0.1' + requires: + - cpan-yaml-syck + diff --git a/libcrange/root/etc/libcrange.conf.example b/libcrange/root/etc/libcrange.conf.example new file mode 100644 index 0000000..ab0633f --- /dev/null +++ b/libcrange/root/etc/libcrange.conf.example @@ -0,0 +1,12 @@ +############ +# +# Basic libcrange config file +# Actual file gemstone managed +# +############ +loadmodule ip +loadmodule yst-ip-list +loadmodule nodescf +perlmodule LibrangeUtils +perlmodule LibrangeAdminscf + diff --git a/libcrange/root/var/libcrange/perl/LibrangeAdminscf.pm b/libcrange/root/var/libcrange/perl/LibrangeAdminscf.pm new file mode 100644 index 0000000..3a262a1 --- /dev/null +++ b/libcrange/root/var/libcrange/perl/LibrangeAdminscf.pm @@ -0,0 +1,73 @@ +package LibrangeAdminscf; + +use Libcrange; +use YAML::Syck; +use warnings 'all'; + +sub functions_provided { + return qw/boot_v v_boot boothosts bh/; +} + +sub _get_cf { + my $rr = shift; + my $path = Libcrange::get_var($rr, "nodescf_path"); + $path ||= "/home/seco/tools/conf"; + my $file = "$path/admins.cf"; + unless (-r $file) { + Libcrange::warn($rr, "$path/admins.cf not readable"); + return; + } + return YAML::Syck::LoadFile($file); +} + +sub boot_v { + my $rr = shift; + my $range = shift; + + my $cf = _get_cf($rr); + return unless $cf; + + my %net_bh; + while ( my ( $bh, $bh_cfg ) = each(%$cf) ) { + for my $net ( @{ $bh_cfg->{networks} } ) { + push @{ $net_bh{$net} }, $bh; + } + } + + my @ret; + for my $net (@$range) { + $net =~ s/\A"(.*)"\z/$1/s; + push @ret, @{ $net_bh{$net} }; + } + return @ret; +} + +sub v_boot { + my $rr = shift; + my $range = shift; + my $cf = _get_cf(); + return unless $cf; + + my @ret; + for my $bh (@$range) { + my $bh_cfg = $cf->{$bh}; + next unless $bh_cfg; + + push @ret, map { qq("$_") } @{ $bh_cfg->{networks} }; + } + return @ret; +} + +sub boothosts { + my $rr = shift; + my $cf = _get_cf(); + return keys %$cf; +} + +sub bh { + my ($rr, $range) = @_; + my $what = join( ",", @{$range} ); + return Libcrange::expand( $rr, "boot_v(vlan($what))" ); +} + +1; diff --git a/libcrange/root/var/libcrange/perl/LibrangeCentcom.pm b/libcrange/root/var/libcrange/perl/LibrangeCentcom.pm new file mode 100644 index 0000000..3ba6169 --- /dev/null +++ b/libcrange/root/var/libcrange/perl/LibrangeCentcom.pm @@ -0,0 +1,56 @@ +package LibrangeCentcom; + +sub functions_provided { + return qw/centcom_environment centcom_services centcom_facility + centcom_status/; +} + +sub centcom_status { + eval "require Centcom;"; + return () if ($@); + my ( $rr, $range ) = @_; + my @ret; + my $centcom = Centcom->new; + for my $status (@$range) { + push @ret, $centcom->findHosts( { status => $status } ); + } + return @ret; +} + +sub centcom_facility { + eval "require Centcom;"; + return () if ($@); + my ( $rr, $range ) = @_; + my @ret; + my $centcom = Centcom->new; + for my $env (@$range) { + push @ret, $centcom->findHosts( { facility => $env } ); + } + return @ret; +} + +sub centcom_services { + eval "require Centcom;"; + return () if ($@); + my ( $rr, $range ) = @_; + my @ret; + my $centcom = Centcom->new; + for my $env (@$range) { + push @ret, $centcom->findHosts( { services => $env } ); + } + return @ret; +} + +sub centcom_environment { + eval "require Centcom;"; + return () if ($@); + my ( $rr, $range ) = @_; + my @ret; + my $centcom = Centcom->new; + for my $env (@$range) { + push @ret, $centcom->findHosts( { environment => $env } ); + } + return @ret; +} + +1; diff --git a/libcrange/root/var/libcrange/perl/LibrangeOpsdb.pm b/libcrange/root/var/libcrange/perl/LibrangeOpsdb.pm new file mode 100644 index 0000000..fb20fbc --- /dev/null +++ b/libcrange/root/var/libcrange/perl/LibrangeOpsdb.pm @@ -0,0 +1,34 @@ +package LibrangeOpsdb; + +use Seco::OpsDB; + +sub functions_provided { + Seco::OpsDB->connect; + return qw/order orders/; +} + +sub orders { + my @result; + my $it = Seco::OpsDB::NodesGroup->search_where( gtype => 'groups' ); + while ( my $order = $it->next ) { + push @result, $order->name; + } + return @result; +} + +sub order { + my ( $rr, $r_orders ) = @_; + my @result; + for my $order (@$r_orders) { + my $it = Seco::OpsDB::NodesGroup->retrieve( + gname => $order, + gtype => 'groups' + ); + while ( my $node = $it->next ) { + push @result, $node->name; + } + } + return @result; +} + +1 diff --git a/libcrange/root/var/libcrange/perl/LibrangeSS.pm b/libcrange/root/var/libcrange/perl/LibrangeSS.pm new file mode 100644 index 0000000..704c21d --- /dev/null +++ b/libcrange/root/var/libcrange/perl/LibrangeSS.pm @@ -0,0 +1,86 @@ +package LibrangeSS; + +sub functions_provided { + return qw/ ds_rid flatten flatten_all /; +} + +sub _get_seco_root { + my $SECO_ROOT = "/home/seco/tools/conf/"; + return $SECO_ROOT; +} + +sub ds_rid { + eval "require YAML::Syck"; + return () if ($@); + + my $rr = shift; + my $range = shift; + return "one node at a time" if scalar @$range != 1; + + my $SECO_ROOT = _get_seco_root(); + + my $node = pop @$range; + my $cluster = Libcrange::expand($rr, "clusters(clusters(clusters($node)))"); + + my $rid_file = $SECO_ROOT . $cluster . '/rid.yaml'; + my $rid = YAML::Syck::LoadFile($rid_file); + + return $rid->{$node}; +} + +sub rid_map { + my $rr = shift; + my $range = shift; + return "one cluster at a time" if scalar @$range != 1; + + my $SECO_ROOT = _get_seco_root(); + + my $cluster = pop @$range; + my $rid_file = $SECO_ROOT . $cluster . '/rid.yaml'; + + open $fh, "$rid_file" or return "can't open rid file"; + my $contents = join '', <$fh>; + close $fh; + + return $contents; +} + +sub flatten { + my $rr = shift; + my $range = shift; + + my @ret = (); + + for my $elem (@{ $range }){ + if (my @expansion = Libcrange::expand($rr, "%$elem") ){ + push @ret, flatten( $rr, [ $_ ] ) for @expansion; + } + else { + push @ret, $elem; + } + } + + return @ret; +} + +sub flatten_all { + my $rr = shift; + my $range = shift; + + use Data::Dumper; + warn Dumper $range; + + my @ret = (); + + for my $elem (@{ $range }){ + if (my @expansion = Libcrange::expand($rr, "%$elem") ){ + push @ret, flatten_all( $rr, [ $_ . ":ALL" ] ) for @expansion; + } + else { + push @ret, $elem; + } + } + + return @ret; +} +1; diff --git a/libcrange/root/var/libcrange/perl/LibrangeUtils.pm b/libcrange/root/var/libcrange/perl/LibrangeUtils.pm new file mode 100644 index 0000000..643d724 --- /dev/null +++ b/libcrange/root/var/libcrange/perl/LibrangeUtils.pm @@ -0,0 +1,37 @@ +package LibrangeUtils; + +sub functions_provided { + return qw/clean limit count even odd/; +} + +sub clean { + my ( $rr, $range ) = @_; + my @result = map { s/\.inktomisearch\.com//; $_ } @$range; + return @result; +} + +sub count { + my ( $rr, $range ) = @_; + return scalar @$range; +} + +sub limit { + my ( $rr, $r_limit, $range ) = @_; + my $limit = $r_limit->[0]; + my @range = @$range; + return @range[ 0 .. ( $limit - 1 ) ]; +} + +sub even { + my ( $rr, $range ) = @_; + my @result = grep { /[02468]\z/ms } @$range; + return @result; +} + +sub odd { + my ( $rr, $range ) = @_; + my @result = grep { /[13579](?:\.inktomisearch\.com)?\z/ms } @$range; + return @result; +} + +1; diff --git a/libcrange/scripts/build b/libcrange/scripts/build new file mode 100755 index 0000000..50610dc --- /dev/null +++ b/libcrange/scripts/build @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e +set -x + +aclocal || exit 1 +libtoolize --force || exit 1 +automake-1.9 -a || exit 1 +autoconf || exit 1 +./configure || exit 1 +make || exit 1 +make install || exit 1 +cd perl +sh ./build || exit 1 + diff --git a/libcrange/scripts/post.sh b/libcrange/scripts/post.sh new file mode 100755 index 0000000..63dc528 --- /dev/null +++ b/libcrange/scripts/post.sh @@ -0,0 +1,6 @@ +#!/bin/sh +/sbin/ldconfig +if [ -r /etc/httpd/conf.d/mod_range.conf ]; then + /etc/init.d/httpd restart +fi +exit 0 diff --git a/libcrange/scripts/ybuild b/libcrange/scripts/ybuild new file mode 100755 index 0000000..ce6c661 --- /dev/null +++ b/libcrange/scripts/ybuild @@ -0,0 +1,6 @@ +#!/bin/sh + +rm -rf ../BUILD +export PATH="/home/y/bin:$PATH" +DESTDIR=$PWD/../BUILD CFLAGS=" -DYAHOO_SOURCE " ../scripts/build + diff --git a/libcrange/source/AUTHORS b/libcrange/source/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/ChangeLog b/libcrange/source/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/INSTALL b/libcrange/source/INSTALL new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/Makefile.am b/libcrange/source/Makefile.am new file mode 100644 index 0000000..fcff9e6 --- /dev/null +++ b/libcrange/source/Makefile.am @@ -0,0 +1,3 @@ +EXTRA_DIST = reconf configure +SUBDIRS= src functions + diff --git a/libcrange/source/NEWS b/libcrange/source/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/README b/libcrange/source/README new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/config.h.in b/libcrange/source/config.h.in new file mode 100644 index 0000000..f87bc54 --- /dev/null +++ b/libcrange/source/config.h.in @@ -0,0 +1,98 @@ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#undef HAVE_DECL_STRERROR_R + +/* Define to 1 if you have the header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_INTTYPES_H + +/* Define to 1 if you have the `crypt' library (-lcrypt). */ +#undef HAVE_LIBCRYPT + +/* Define to 1 if you have the `m' library (-lm). */ +#undef HAVE_LIBM + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#undef HAVE_LIBPTHREAD + +/* Define to 1 if you have the `z' library (-lz). */ +#undef HAVE_LIBZ + +/* Define to 1 if you have the header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if `stat' has the bug that it succeeds when given the + zero-length file name argument. */ +#undef HAVE_STAT_EMPTY_STRING_BUG + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STDLIB_H + +/* Define to 1 if you have the `strerror' function. */ +#undef HAVE_STRERROR + +/* Define to 1 if you have the `strerror_r' function. */ +#undef HAVE_STRERROR_R + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRING_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_STAT_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TYPES_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if `lstat' dereferences a symlink specified with a trailing + slash. */ +#undef LSTAT_FOLLOWS_SLASHED_SYMLINK + +/* Package name */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define to 1 if strerror_r returns char *. */ +#undef STRERROR_R_CHAR_P + +/* Package version */ +#undef VERSION + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +#undef YYTEXT_POINTER + +/* Define to empty if `const' does not conform to ANSI C. */ +#undef const + +/* Define to `unsigned' if does not define. */ +#undef size_t diff --git a/libcrange/source/configure.ac b/libcrange/source/configure.ac new file mode 100644 index 0000000..afc8a68 --- /dev/null +++ b/libcrange/source/configure.ac @@ -0,0 +1,117 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.59) +AC_INIT(libcrange, 1.0, prod-eng@yahoo-inc.com) +AC_CONFIG_SRCDIR(src/libcrange.h) +AM_INIT_AUTOMAKE +AM_CONFIG_HEADER([config.h]) + +AC_DEFINE_UNQUOTED(PACKAGE, $PACKAGE, [Package name]) +AC_DEFINE_UNQUOTED(VERSION, $VERSION, [Package version]) + +# Checks for programs. +AC_PROG_YACC +AM_PROG_LEX +AC_PROG_CC +AC_LANG_C +AC_PROG_INSTALL +AC_PROG_MAKE_SET +AC_PROG_RANLIB +AC_PROG_LIBTOOL + +perl_default="/usr/local/bin/perl" +AC_ARG_ENABLE([perl], AC_HELP_STRING([--enable-perl=PERL], + [Use this perl binary to build libcrange's perl + functionality. + (Default = /usr/local/bin/perl)]), + PERL=$enable_perl,PERL=$perl_default) +AC_DEFUN([AC_CHECK_PERL],[ +AC_MSG_CHECKING(perl in $PERL) +if test [ -f "$PERL" ] +then + AC_SUBST(PERL) + AC_MSG_RESULT(yes) + AC_MSG_CHECKING(perl includes) + PERL_CFLAGS=`${PERL} -MExtUtils::Embed -e ccopts` + AC_SUBST(PERL_CFLAGS) + AC_MSG_RESULT($PERL_CFLAGS) + AC_MSG_CHECKING(perl libraries) + PERL_LIBS=`${PERL} -MExtUtils::Embed -e ldopts` + AC_SUBST(PERL_LIBS) + AC_MSG_RESULT($PERL_LIBS) +else + AC_MSG_ERROR([${PERL} not found]) +fi +]) + +AC_DEFUN([AC_CHECK_PCRE],[ +AC_PATH_PROG(pcreconfig,pcre-config) +if test [ -z "$pcreconfig" ] +then + AC_MSG_ERROR([pcre-config executable not found]) +else + AC_MSG_CHECKING(pcre includes) + PCRE_CFLAGS=`${pcreconfig} --cflags` + AC_MSG_RESULT($PCRE_CFLAGS) + AC_SUBST(PCRE_CFLAGS) + AC_MSG_CHECKING(pcre libraries) + PCRE_LIBS=`${pcreconfig} --libs` + AC_MSG_RESULT($PCRE_LIBS) + AC_SUBST(PCRE_LIBS) +fi +]) + +AC_DEFUN([AC_CHECK_APR],[ +AC_PATH_PROG(aprconfig,apr-config) +if test [ -z "$aprconfig" ] +then + AC_PATH_PROG(aprconfig, apr-1-config) +fi +if test [ -z "$aprconfig" ] +then + AC_MSG_ERROR([apr-config/apr-1-config executable not found]) +else + AC_MSG_CHECKING(apr includes) + APR_CFLAGS=`${aprconfig} --cflags --cppflags --includes` + AC_MSG_RESULT($APR_CFLAGS) + AC_SUBST(APR_CFLAGS) + AC_MSG_CHECKING(apr libraries) + APR_LIBS=`${aprconfig} --link-libtool --libs` + APR_LIBS_LD=`${aprconfig} --link-ld --libs` + AC_MSG_RESULT($APR_LIBS) + AC_SUBST(APR_LIBS) + AC_SUBST(APR_LIBS_LD) +fi +]) + +AC_CHECK_PERL +AC_CHECK_PCRE +AC_CHECK_APR + +AC_CHECK_LIB([crypt], [crypt_r], [], [exit 1]) +AC_CHECK_LIB([m], [sin], [], [exit 1]) +AC_CHECK_LIB([pthread], [pthread_create], [], [exit 1]) +AC_CHECK_LIB([z], [zlibVersion], [], [exit 1]) + +# Checks for header files. +AC_HEADER_STDC +AC_CHECK_HEADERS([stdlib.h string.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST +AC_TYPE_SIZE_T + +# Checks for library functions. +AC_FUNC_STAT +AC_FUNC_STRERROR_R +AC_CHECK_FUNCS([strerror]) + +AC_CONFIG_FILES([Makefile + doc/Makefile + m4/Makefile + src/Makefile + perl/Makefile.PL + perl/build + functions/Makefile]) +AC_OUTPUT diff --git a/libcrange/source/doc/Makefile.am b/libcrange/source/doc/Makefile.am new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/functions/Makefile.am b/libcrange/source/functions/Makefile.am new file mode 100644 index 0000000..70f4f7d --- /dev/null +++ b/libcrange/source/functions/Makefile.am @@ -0,0 +1,12 @@ +AM_CFLAGS = -Wall -DLIBCRANGE_FUNCDIR=\"$(pkglibdir)\" -I../src @PCRE_CFLAGS@ @APR_CFLAGS@ +AM_LDFLAGS = -module -L../src -lcrange @PCRE_LIBS@ @APR_LIBS@ + +pkglib_LTLIBRARIES = yst-ip-list.la ip.la nodescf.la + +nodescf_la_SOURCES = nodescf.c +yst_ip_list_la_SOURCES = yst-ip-list.c netblock.c tinydns_ip.c \ + hosts-netblocks.c +ip_la_SOURCES = ip.c tinydns_ip.c + + + diff --git a/libcrange/source/functions/group-mysql.c b/libcrange/source/functions/group-mysql.c new file mode 100644 index 0000000..266968e --- /dev/null +++ b/libcrange/source/functions/group-mysql.c @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include +#include + +#include "set.h" +#include "libcrange.h" +#include "range.h" + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"group", 0}; + return functions; +} + +range* rangefunc_group(range_request* rr, range** r) +{ + range *ret; + const char **members; + int i; + MYSQL *conn; + MYSQL_ROW row; + MYSQL_RES *res; + apr_pool_t* pool = range_request_pool(rr); + libcrange* lr = range_request_lr(rr); + + ret = range_new(rr); + members = range_get_hostnames(pool, r[0]); + + if(!(conn = (MYSQL *)libcrange_get_cache(lr, "mysql:nodes"))) { + const char* mysql_user = libcrange_getcfg(lr, "mysqluser"); + const char* mysql_db = libcrange_getcfg(lr, "mysqldb"); + const char* mysql_passwd = libcrange_getcfg(lr, "mysqlpasswd"); + + conn = mysql_init(NULL); + mysql_real_connect(conn, "docking", mysql_user, mysql_passwd, + mysql_db, 0, NULL, 0); + libcrange_set_cache(lr, "mysql:nodes", conn); + } + for(i = 0; members[i]; i++) { /* for each gemgroup */ + int all = strcmp(members[i], "ALL") == 0; + + if (all) { + const char* query = "select name from nodes"; + if (mysql_query(conn, query)) { + fprintf(stderr, "query: %s failed: %s\n", + query, mysql_error(conn)); + return range_new(rr); + } + } else { + const char* query = apr_psprintf(pool, + "select range from tags where name='%s'", + members[i]); + if (mysql_query(conn, query)) { + fprintf(stderr, "query: %s failed: %s\n", + query, mysql_error(conn)); + return range_new(rr); + } + } + res = mysql_store_result(conn); + assert(res); + while ((row = mysql_fetch_row(res)) != NULL) { + range* this_group; + const char* result = row[0]; + if (all) { + range_add(ret, result); + } else { + this_group = do_range_expand(rr, result); + set_union_inplace(ret->nodes, this_group->nodes); + } + } + mysql_free_result(res); + } + + return ret; +} diff --git a/libcrange/source/functions/group-sqlite.c b/libcrange/source/functions/group-sqlite.c new file mode 100644 index 0000000..c259289 --- /dev/null +++ b/libcrange/source/functions/group-sqlite.c @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include + +#include "set.h" +#include "libcrange.h" +#include "range.h" + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"group", 0}; + return functions; +} + +#define ALL_NODES_SQL "select name from nodes" +#define RANGE_FROM_TAGS "select range from tags where name=?" + +range* rangefunc_group(range_request* rr, range** r) +{ + range* ret; + const char** members; + int i, err; + sqlite3* db; + sqlite3_stmt* tag_stmt; + sqlite3_stmt* all_nodes_stmt; + apr_pool_t* pool = range_request_pool(rr); + libcrange* lr = range_request_lr(rr); + + ret = range_new(rr); + members = range_get_hostnames(pool, r[0]); + + if (!(db = libcrange_get_cache(lr, "sqlite:nodes"))) { + const char* sqlite_db_path = libcrange_getcfg(lr, "sqlitedb"); + if (!sqlite_db_path) sqlite_db_path = DEFAULT_SQLITE_DB; + + err = sqlite3_open(sqlite_db_path, &db); + if (err != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", sqlite_db_path, sqlite3_errmsg(db)); + return ret; + } + + libcrange_set_cache(lr, "sqlite:nodes", db); + } + + + /* prepare our selects */ + err = sqlite3_prepare(db, ALL_NODES_SQL, strlen(ALL_NODES_SQL), + &all_nodes_stmt, NULL); + if (err != SQLITE_OK) { + fprintf(stderr, "%s: %s\n", ALL_NODES_SQL, sqlite3_errmsg(db)); + abort(); + } + + err = sqlite3_prepare(db, RANGE_FROM_TAGS, strlen(RANGE_FROM_TAGS), + &tag_stmt, NULL); + assert(err == SQLITE_OK); + + /* for each group */ + for (i = 0; members[i]; ++i) { + sqlite3_stmt* stmt; + if (strcmp(members[i], "ALL") == 0) { + stmt = all_nodes_stmt; + } else { + stmt = tag_stmt; + /* bind the current group name */ + sqlite3_bind_text(tag_stmt, 1, members[i], strlen(members[i]), SQLITE_STATIC); + } + + while (sqlite3_step(stmt) == SQLITE_ROW) { + range* this_group; + const char* result = (const char*)sqlite3_column_text(stmt, 0); + if (stmt == all_nodes_stmt) { + range_add(ret, result); + } else { + this_group = do_range_expand(rr, result); + set_union_inplace(ret->nodes, this_group->nodes); + } + } + sqlite3_reset(stmt); + } + sqlite3_finalize(all_nodes_stmt); + sqlite3_finalize(tag_stmt); + + return ret; +} diff --git a/libcrange/source/functions/hosts-netblocks.c b/libcrange/source/functions/hosts-netblocks.c new file mode 100644 index 0000000..67a246a --- /dev/null +++ b/libcrange/source/functions/hosts-netblocks.c @@ -0,0 +1,169 @@ +#include "hosts-netblocks.h" +#include "netblock.h" +#include "set.h" +#include +#include +#include + +#define HOSTS_NETBLOCK_CACHE "yst-ip-list:hosts_netblocks" +#define HOSTS_DC_CACHE "yst-ip-list:hosts_dc" +#define DC_HOSTS_CACHE "yst-ip-list:dc_hosts" +#define NETBLOCK_HOSTS_CACHE "yst-ip-list:netblocks_hosts" + +static void init_caches(libcrange* lr) +{ + const char* default_domain; + apr_pool_t* pool = libcrange_get_pool(lr); + range_request* rr; + + /* Create a netblock -> hosts mapping */ + ip_host** all_hosts = tinydns_all_ip_hosts(lr, pool); + set* hn = set_new(pool, 100000); /* hosts networks */ + set* hd = set_new(pool, 100000); /* hosts datacenter */ + set* nh = set_new(pool, 1500) ; /* network hosts */ + set* dh = set_new(pool, 0); /* datacenter hosts */ + + assert(all_hosts); + rr = range_request_new(lr, pool); + default_domain = libcrange_get_default_domain(lr); + while (*all_hosts) { + char short_hostname[512]; + int len_hostname; + char net_key[32]; + set_element* elt; + range* r; + ip_host* iph = *all_hosts++; + + const netblock* block; + const net_colo* nc = netcolo_for_ip(lr, iph->ip); + if (!nc) continue; + + strncpy(short_hostname, iph->hostname, sizeof short_hostname); + short_hostname[sizeof short_hostname - 1] = '\0'; + len_hostname = strlen(short_hostname); + if (short_hostname[len_hostname - 1] == '.') + /* remove '.' at the end adjusting len */ + short_hostname[--len_hostname] = '\0'; + + if (default_domain) { + /* remove default_domain */ + int len_domain = strlen(default_domain); + if (len_hostname > len_domain) { + if (strcmp(&short_hostname[len_hostname - len_domain], + default_domain) == 0) + short_hostname[len_hostname - len_domain - 1] = '\0'; + } + } + + block = nc->net; + set_add(hn, short_hostname, (void*)block); + set_add(hn, iph->ip->str, (void*)block); + set_add(hd, short_hostname, (void*)(nc->colo)); + set_add(hd, iph->ip->str, (void*)(nc->colo)); + + elt = set_get(nh, netblock_key(block, net_key, sizeof net_key)); + if (!elt) { + r = range_new(rr); + set_add(nh, net_key, r); + } + else + r = elt->data; + range_add(r, short_hostname); + + elt = set_get(dh, nc->colo); + if (!elt) { + r = range_new(rr); + set_add(dh, nc->colo, r); + } + else + r = elt->data; + range_add(r, short_hostname); + } + libcrange_set_cache(lr, HOSTS_NETBLOCK_CACHE, hn); + libcrange_set_cache(lr, NETBLOCK_HOSTS_CACHE, nh); + libcrange_set_cache(lr, DC_HOSTS_CACHE, dh); + libcrange_set_cache(lr, HOSTS_DC_CACHE, hd); +} + +static set* hosts_netblocks(libcrange* lr) +{ + set* hn = libcrange_get_cache(lr, HOSTS_NETBLOCK_CACHE); + if (!hn) { + init_caches(lr); + hn = libcrange_get_cache(lr, HOSTS_NETBLOCK_CACHE); + } + return hn; +} + +static set* netblock_hosts(libcrange* lr) +{ + set* nh = libcrange_get_cache(lr, NETBLOCK_HOSTS_CACHE); + if (!nh) { + init_caches(lr); + nh = libcrange_get_cache(lr, NETBLOCK_HOSTS_CACHE); + } + return nh; +} + +static set* datacenter_hosts(libcrange* lr) +{ + set* dh = libcrange_get_cache(lr, DC_HOSTS_CACHE); + if (!dh) { + init_caches(lr); + dh = libcrange_get_cache(lr, DC_HOSTS_CACHE); + } + return dh; +} + +static set* hosts_dc(libcrange* lr) +{ + set* hd = libcrange_get_cache(lr, HOSTS_DC_CACHE); + if (!hd) { + init_caches(lr); + hd = libcrange_get_cache(lr, HOSTS_DC_CACHE); + } + return hd; +} + +range* hosts_in_netblock(range_request* rr, const char* netblock_key) +{ + libcrange* lr = range_request_lr(rr); + set* nh = netblock_hosts(lr); + set_element* elt = set_get(nh, netblock_key); + if (elt) + return elt->data; + + range_request_warn_type(rr, "NETBLOCK_NOT_FOUND", netblock_key); + return range_new(rr); +} + +range* hosts_in_dc(range_request* rr, const char* dc) +{ + libcrange* lr = range_request_lr(rr); + set* dh = datacenter_hosts(lr); + set_element* elt = set_get(dh, dc); + if (elt) + return elt->data; + + range_request_warn_type(rr, "DC_NOT_FOUND", dc); + return range_new(rr); +} + +const netblock* netblock_for_host(range_request* rr, const char* host) +{ + libcrange* lr = range_request_lr(rr); + set* hn = hosts_netblocks(lr); + set_element* block = set_get(hn, host); + + if (block) return block->data; + return NULL; +} + +const char* dc_for_host(range_request* rr, const char* host) +{ + libcrange* lr = range_request_lr(rr); + set* hd = hosts_dc(lr); + set_element* dc = set_get(hd, host); + if (dc) return dc->data; + return NULL; +} diff --git a/libcrange/source/functions/hosts-netblocks.h b/libcrange/source/functions/hosts-netblocks.h new file mode 100644 index 0000000..d6b1253 --- /dev/null +++ b/libcrange/source/functions/hosts-netblocks.h @@ -0,0 +1,13 @@ +#ifndef HOSTS_NETBLOCKS_H +#define HOSTS_NETBLOCKS_H + +#include "libcrange.h" +#include "range.h" +#include "netblock.h" + +range* hosts_in_netblock(range_request* rr, const char* netblock_key); +const netblock* netblock_for_host(range_request* rr, const char* host); +range* hosts_in_dc(range_request* rr, const char* dc); +const char* dc_for_host(range_request* rr, const char* host); + +#endif diff --git a/libcrange/source/functions/ip.c b/libcrange/source/functions/ip.c new file mode 100644 index 0000000..5b7ed15 --- /dev/null +++ b/libcrange/source/functions/ip.c @@ -0,0 +1,29 @@ +#include "libcrange.h" +#include "range.h" +#include "tinydns_ip.h" + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"ip", 0}; + return functions; +} + +range* rangefunc_ip(range_request* rr, range** r) +{ + range* ret; + const char** members; + int i; + ip* ip; + apr_pool_t* pool = range_request_pool(rr); + + ret = range_new(rr); + members = range_get_hostnames(pool, r[0]); + for (i = 0; members[i]; i++) { + ip = tinydns_get_ip(rr, members[i]); + if (ip) + range_add(ret, ip->str); + else + range_request_warn_type(rr, "NOTINYDNS", members[i]); + } + return ret; +} diff --git a/libcrange/source/functions/netblock.c b/libcrange/source/functions/netblock.c new file mode 100644 index 0000000..22d30cb --- /dev/null +++ b/libcrange/source/functions/netblock.c @@ -0,0 +1,219 @@ +#include "netblock.h" +#include "set.h" +#include +#include +#include +#include +#include +#include +#include + +#define IMASK(n) (0 - (1 << (32 - n))) +#define NETMASK_RE "^\"?(\\d+\\.\\d+\\.\\d+\\.\\d+)/(\\d+)\"?$" +#define IP_RE "^\\d+\\.\\d+\\.\\d+\\.\\d+$" +#define TWO_FIELDS_RE "^(\\S+)\\s+(\\S+)" + +#define NET_COLO_CACHE "net:netkey-netcolo" +#define COLO_NET_CACHE "net:colo-netblock" + +static pcre* netmask_re = 0; +static pcre* ip_re = 0; +static pcre* two_fields_re = 0; + +static void compile_regexes() +{ + if (!netmask_re) { + int err_offset; + const char* error; + netmask_re = pcre_compile(NETMASK_RE, 0, &error, &err_offset, NULL); + assert(netmask_re); + + ip_re = pcre_compile(IP_RE, 0, &error, &err_offset, NULL); + assert(ip_re); + + two_fields_re = pcre_compile(TWO_FIELDS_RE, 0, &error, &err_offset, NULL); + assert(two_fields_re); + } +} + +static set* read_netblocks(libcrange* lr) +{ + set* result; + set* colo_nets; + + FILE* fp; + char line[4096]; + int line_no; + apr_pool_t* pool; + range_request* rr; + + if ((result = libcrange_get_cache(lr, NET_COLO_CACHE)) != 0) + return result; + + pool = libcrange_get_pool(lr); + compile_regexes(); + result = set_new(pool, 0); + colo_nets = set_new(pool, 0); + + fp = fopen(YST_IP_LIST, "r"); + if (!fp) { + fprintf(stderr, "%s: %s", YST_IP_LIST, + strerror(errno)); + return set_new(pool, 0); + } + + line_no = 0; + rr = range_request_new(lr, pool); + while (fgets(line, sizeof line, fp)) { + int count; + int ovector[30]; + int n; + char* p = line; + const char* colo; + const char* netblock_str; + char* key; + set_element* elt; + range* r; + netblock* net; + net_colo* nc = apr_palloc(pool, sizeof(net_colo)); + + ++line_no; + p = line; + while (*p && isspace(*p)) ++p; + if (*p == '#') continue; + n = strlen(p); + if (p[n - 1] != '\n') { + fprintf(stderr, "%s: line %d is too long.\n", + YST_IP_LIST, line_no); + fclose(fp); + return set_new(pool, 0); + } + count = pcre_exec(two_fields_re, NULL, p, n, + 0, 0, ovector, 30); + if (count < 3) continue; + + colo = &p[ovector[2]]; + p[ovector[3]] = '\0'; + + netblock_str = &p[ovector[4]]; + p[ovector[5]] = '\0'; + + net = netblock_from_string(pool, netblock_str); + nc->net = net; + nc->colo = apr_pstrdup(pool, colo); + + key = apr_psprintf(pool, "%x/%d", net->base, net->bits); + set_add(result, key, nc); + elt = set_get(colo_nets, colo); + if (!elt) { + r = range_new(rr); + r->quoted = 1; + set_add(colo_nets, colo, r); + } + else + r = elt->data; + range_add(r, net->str); + } + fclose(fp); + + libcrange_set_cache(lr, COLO_NET_CACHE, colo_nets); + libcrange_set_cache(lr, NET_COLO_CACHE, result); + return result; +} + +net_colo* netcolo_for_ip(libcrange* lr, const ip* node_ip) +{ + int bits; + set* netblocks; + unsigned bin_ip; + + assert(lr); + assert(node_ip); + netblocks = read_netblocks(lr); + bin_ip = node_ip->binary; + for (bits=32; bits > 0; --bits) { + char netmask[32]; + set_element* elt; + unsigned base = bin_ip & IMASK(bits); + sprintf(netmask, "%x/%d", base, bits); + elt = set_get(netblocks, netmask); + if (elt) + return elt->data; + } + return NULL; +} + +char* netblock_key(const netblock* block, char* buf, size_t n) +{ + snprintf(buf, n, "%x/%d", block->base, block->bits); + buf[n - 1] = '\0'; + return buf; +} + +const char* netblock_to_str(const netblock* block) +{ + assert(block); + return block->str; +} + +netblock* netblock_from_string(apr_pool_t* pool, const char* netmask) +{ + int count; + int ovector[30]; + const char* base; + const char* bits; + int len = strlen(netmask); + netblock* result = apr_palloc(pool, sizeof(netblock)); + + compile_regexes(); + count = pcre_exec(netmask_re, NULL, netmask, len, + 0, 0, ovector, 30); + if (count > 0) { + pcre_get_substring(netmask, ovector, count, 1, &base); + pcre_get_substring(netmask, ovector, count, 2, &bits); + + result->base = str2ip(base); + result->bits = atoi(bits); + + result->str = apr_pstrdup(pool, netmask); + } + else { + char* repr = apr_palloc(pool, len + 4); + count = pcre_exec(ip_re, NULL, netmask, len, + 0, 0, ovector, 30); + /* FIXME: deal with errors */ + if (count <= 0) { + printf("ERROR: [%s] trying to parse a netblock\n", netmask); + abort(); + } + + result->bits = 32; + pcre_get_substring(netmask, ovector, count, 0, &base); + result->base = str2ip(base); + pcre_free_substring(base); + + strcpy(repr, netmask); + strcat(repr, "/32"); + result->str = repr; + } + + return result; +} + +range* netblocks_for_dc(range_request* rr, const char* dc) +{ + set* colo_nets; + set_element* elt; + libcrange* lr = range_request_lr(rr); + + read_netblocks(lr); /* make sure the caches are up-to-date */ + + colo_nets = libcrange_get_cache(lr, COLO_NET_CACHE); + elt = set_get(colo_nets, dc); + if (elt) + return elt->data; + else { + range_request_warn_type(rr, "NO_DC", dc); + return range_new(rr); + } +} diff --git a/libcrange/source/functions/netblock.h b/libcrange/source/functions/netblock.h new file mode 100644 index 0000000..158d511 --- /dev/null +++ b/libcrange/source/functions/netblock.h @@ -0,0 +1,30 @@ +#ifndef NETBLOCK_H +#define NETBLOCK_H + +#include "libcrange.h" +#include "range.h" +#include "tinydns_ip.h" + +#define YST_IP_LIST "/etc/yst-ip-list" + +typedef struct netblock +{ + unsigned base; + int bits; + const char* str; +} netblock; + +typedef struct net_colo +{ + netblock* net; + const char* colo; +} net_colo; + +net_colo* netcolo_for_ip(libcrange* lr, const ip* node_ip); +const char* netblock_to_str(const netblock* block); +netblock* netblock_from_string(apr_pool_t* pool, const char* netmask); +char* netblock_key(const netblock* block, char* buf, size_t n); + +range* netblocks_for_dc(range_request* lr, const char* dc); + +#endif diff --git a/libcrange/source/functions/nodescf.c b/libcrange/source/functions/nodescf.c new file mode 100644 index 0000000..64a3a73 --- /dev/null +++ b/libcrange/source/functions/nodescf.c @@ -0,0 +1,809 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "libcrange.h" +#include "range.h" + +static const char* nodescf_path = "/home/seco/tools/conf"; + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"mem", "cluster", "clusters", + "group", "get_admin", "get_cluster", + "get_groups", "has", "allclusters", 0 }; + + /* initialize our path to the nodes database */ + + /* First try env variable */ + const char* altpath = getenv( "LIBCRANGE_NODESCF_PATH" ); + + /* Next try variable from config file */ + if( ! altpath ) + altpath = libcrange_getcfg(lr, "nodescf_path"); + + /* If no alternative was specified, keep the default */ + if (altpath) + nodescf_path = altpath; + + return functions; +} + + +typedef struct cache_entry +{ + time_t mtime; + apr_pool_t* pool; + set* sections; +} cache_entry; + +static set* _get_ignore_set(range_request* rr) +{ + char line[32768]; + apr_pool_t* pool = range_request_pool(rr); + const char* ignore_path = apr_psprintf(pool, "%s/all/IGNORE", nodescf_path); + set* ret = set_new(pool, 0); + FILE* fp = fopen(ignore_path, "r"); + if (!fp) return ret; + + while (fgets(line, sizeof line, fp) != NULL) { + int len; + char* p; + line[sizeof line - 1] = '\0'; + len = strlen(line); + + if (!len) continue; + if (line[len - 1] == '\n') + line[--len] = '\0'; + + for (p = line; *p; ++p) + if (*p == '#') { + *p = '\0'; + len = strlen(line); + break; + } + + if (!len) continue; + + for (p = &line[len - 1]; isspace(*p); --p) { + --len; + *p = '\0'; + } + if (len <= 0) continue; + + set_add(ret, line, 0); + } + fclose(fp); + + return ret; +} + +#define INCLUDE_RE "^\\s+INCLUDE\\s+(.+)" +#define EXCLUDE_RE "^\\s+EXCLUDE\\s+(.+)" + +static pcre* include_re = NULL; +static pcre* exclude_re = NULL; + +static char* _substitute_dollars(apr_pool_t* pool, + const char* cluster, const char* line) +{ + static char buf[262144]; + char* dst = buf; + int len = strlen(cluster); + int in_regex = 0; + char c; + assert(line); + assert(cluster); + + while ((c = *line) != '\0') { + if (!in_regex && c == '$') { + strcpy(dst, "cluster("); + dst += sizeof("cluster(") - 1; + strcpy(dst, cluster); + dst += len; + *dst++ = ':'; + c = *++line; + while (isalnum(c) || c == '_') { + *dst++ = c; + c = *++line; + } + *dst++ = ')'; + } + else if (c == '/') { + in_regex = !in_regex; + *dst++ = *line++; + } + else { + *dst++ = *line++; + } + } + *dst = '\0'; + return buf; +} + +char* _join_elements(apr_pool_t* pool, char sep, set* the_set) +{ + set_element** members = set_members(the_set); + set_element** p = members; + int total; + char* result; + char* p_res; + + total = 0; + while (*p) { + total += strlen((*p)->name) + 1; + ++p; + } + if (!total) return NULL; + + p = members; + p_res = result = apr_palloc(pool, total); + while (*p) { + int len = strlen((*p)->name); + strcpy(p_res, (*p)->name); + ++p; + p_res += len; + *p_res++ = sep; + } + + *--p_res = '\0'; + return result; +} + +static pcre* vips_re = NULL; + +typedef struct vips +{ + time_t mtime; + apr_pool_t* pool; + set* vips; + set* viphosts; +} vips; + +static vips* _empty_vips(range_request* rr) +{ + apr_pool_t* pool = range_request_pool(rr); + vips* v = apr_palloc(pool, sizeof(*v)); + v->vips = v->viphosts = set_new(pool, 0); + return v; +} + +static vips* _parse_cluster_vips(range_request* rr, const char* cluster) +{ + struct stat st; + int ovector[30]; + char line[32768]; + int line_no; + FILE* fp; + apr_pool_t* req_pool = range_request_pool(rr); + apr_pool_t* lr_pool = range_request_lr_pool(rr); + libcrange* lr = range_request_lr(rr); + set* cache = libcrange_get_cache(lr, "nodescf:cluster_vips"); + const char* vips_path = apr_psprintf(req_pool, "%s/%s/vips.cf", + nodescf_path, cluster); + vips* v; + + if (!cache) { + cache = set_new(lr_pool, 0); + libcrange_set_cache(lr, "nodescf:cluster_vips", cache); + } + + if (stat(vips_path, &st) == -1) { + range_request_warn_type(rr, "NOVIPS", cluster); + return _empty_vips(rr); + } + + v = set_get_data(cache, vips_path); + if (!v) { + v = apr_palloc(lr_pool, sizeof(struct vips)); + apr_pool_create(&v->pool, lr_pool); + v->vips = set_new(v->pool, 0); + v->viphosts = set_new(v->pool, 0); + v->mtime = st.st_mtime; + set_add(cache, vips_path, v); + } + else { + time_t cached_mtime = v->mtime; + if (cached_mtime != st.st_mtime) { + apr_pool_clear(v->pool); + v->vips = set_new(v->pool, 0); + v->viphosts = set_new(v->pool, 0); + v->mtime = st.st_mtime; + } + else /* current cached copy is good */ + return v; + } + + /* create / update the current cached copy */ + fp = fopen(vips_path, "r"); + if (!fp) { + range_request_warn_type(rr, "NOVIPS", cluster); + return _empty_vips(rr); + } + + if (!vips_re) { + const char* error; + vips_re = pcre_compile("^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s*$", 0, &error, + ovector, NULL); + assert(vips_re); + } + + line_no = 0; + while (fgets(line, sizeof line, fp)) { + int len; + int count; + char* p; + + line_no++; + line[sizeof line - 1] = '\0'; + len = strlen(line); + if (len+1 >= sizeof(line) && line[len - 1] != '\n') { + /* incomplete line */ + fprintf(stderr, "%s:%d lines > 32767 chars not supported\n", vips_path, line_no); + exit(-1); + } + + line[--len] = '\0'; /* get rid of the \n */ + for (p = line; *p; ++p) + if (*p == '#') { + *p = '\0'; + break; + } + + len = strlen(line); + if (len == 0) continue; + + for (p = &line[len - 1]; isspace(*p); --p) { + *p = '\0'; + --len; + } + + if (!*line) continue; + + /* 68.142.248.161 as301000 eth0:1 */ + count = pcre_exec(vips_re, NULL, line, len, 0, 0, ovector, 30); + if (count == 4) { + line[ovector[3]] = '\0'; + line[ovector[5]] = '\0'; + line[ovector[7]] = '\0'; + + set_add(v->vips, &line[ovector[2]], 0); + set_add(v->viphosts, &line[ovector[4]], 0); + } + } + + fclose(fp); + return v; +} + +static range* _cluster_vips(range_request* rr, const char* cluster) +{ + vips* v = _parse_cluster_vips(rr, cluster); + return range_from_set(rr, v->vips); +} + +static range* _cluster_viphosts(range_request* rr, const char* cluster) +{ + vips* v = _parse_cluster_vips(rr, cluster); + return range_from_set(rr, v->viphosts); +} + +static set* _cluster_keys(range_request* rr, apr_pool_t* pool, + const char* cluster, const char* cluster_file) +{ + char line[32768]; + char* p; + int ovector[30]; + apr_array_header_t* working_range; + set* sections; + char* section; + char* cur_section; + apr_pool_t* req_pool = range_request_pool(rr); + int line_no; + FILE* fp = fopen(cluster_file, "r"); + + if (!fp) { + range_request_warn(rr, "%s: %s not readable", cluster, cluster_file); + return set_new(pool, 0); + } + + if (!include_re) { + const char* error; + include_re = pcre_compile(INCLUDE_RE, 0, &error, ovector, NULL); + assert(include_re); + + exclude_re = pcre_compile(EXCLUDE_RE, 0, &error, ovector, NULL); + assert(exclude_re); + } + + sections = set_new(pool, 0); + section = cur_section = NULL; + + + working_range = apr_array_make(req_pool, 1, sizeof(char*)); + line_no = 0; + while (fgets(line, sizeof line, fp)) { + int len; + int count; + line_no++; + line[sizeof line - 1] = '\0'; + len = strlen(line); + if (len+1 >= sizeof(line) && line[len - 1] != '\n') { + /* incomplete line */ + fprintf(stderr, "%s:%d lines > 32767 chars not supported\n", cluster_file, line_no); + exit(-1); + } + + line[--len] = '\0'; /* get rid of the \n */ + for (p = line; *p; ++p) + if (*p == '#') { + *p = '\0'; + break; + } + + len = strlen(line); + if (len == 0) continue; + + for (p = &line[len - 1]; isspace(*p); --p) { + *p = '\0'; + --len; + } + + if (!*line) continue; + + if (!(isspace(*line))) { + cur_section = apr_pstrdup(pool, line); + continue; + } + + if (section && strcmp(cur_section, section) != 0) { + set_add(sections, section, + apr_array_pstrcat(pool, working_range, ',')); + working_range = apr_array_make(req_pool, 1, sizeof(char*)); + } + + section = cur_section; + count = pcre_exec(include_re, NULL, line, len, + 0, 0, ovector, 30); + if (count > 0) { + line[ovector[3]] = '\0'; + *(char**)apr_array_push(working_range) = + apr_psprintf(pool, "(%s)", + _substitute_dollars(pool, cluster, &line[ovector[2]])); + continue; + } + + count = pcre_exec(exclude_re, NULL, line, len, + 0, 0, ovector, 30); + if (count > 0) { + line[ovector[3]] = '\0'; + *(char**)apr_array_push(working_range) = + apr_psprintf(pool, "-(%s)", + _substitute_dollars(pool, cluster, &line[ovector[2]])); + } + + } + fclose(fp); + + if (cur_section) + set_add(sections, cur_section, + apr_array_pstrcat(pool, working_range, ',')); + + set_add(sections, "KEYS", _join_elements(pool, ',', sections)); + set_add(sections, "UP", set_get_data(sections, "CLUSTER")); + if (set_get(sections, "ALL") && set_get(sections, "CLUSTER")) + set_add(sections, "DOWN", + apr_psprintf(pool, "(%s)-(%s)", + (char*)set_get_data(sections, "ALL"), + (char*)set_get_data(sections, "CLUSTER"))); + return sections; +} + +static range* _expand_cluster(range_request* rr, + const char* cluster, const char* section) +{ + struct stat st; + const char* res; + libcrange* lr = range_request_lr(rr); + set* cache = libcrange_get_cache(lr, "nodescf:cluster_keys"); + apr_pool_t* req_pool = range_request_pool(rr); + apr_pool_t* lr_pool = range_request_lr_pool(rr); + + const char* cluster_file; + cache_entry* e; + + if (strcmp(section, "VIPS") == 0) + return _cluster_vips(rr, cluster); + + if (strcmp(section, "VIPHOSTS") == 0) + return _cluster_viphosts(rr, cluster); + + cluster_file = apr_psprintf(req_pool, "%s/%s/nodes.cf", nodescf_path, cluster); + if (!cache) { + cache = set_new(lr_pool, 0); + libcrange_set_cache(lr, "nodescf:cluster_keys", cache); + } + + if (stat(cluster_file, &st) == -1) { + range_request_warn_type(rr, "NOCLUSTERDEF", cluster); + return range_new(rr); + } + + e = set_get_data(cache, cluster_file); + if (!e) { + e = apr_palloc(lr_pool, sizeof(struct cache_entry)); + apr_pool_create(&e->pool, lr_pool); + e->sections = _cluster_keys(rr, e->pool, cluster, cluster_file); + e->mtime = st.st_mtime; + set_add(cache, cluster_file, e); + } + else { + time_t cached_mtime = e->mtime; + if (cached_mtime != st.st_mtime) { + apr_pool_clear(e->pool); + e->sections = _cluster_keys(rr, e->pool, cluster, cluster_file); + e->mtime = st.st_mtime; + } + } + + res = set_get_data(e->sections, section); + + if (!res) { + char* cluster_section = apr_psprintf(req_pool, + "%s:%s", cluster, section); + range_request_warn_type(rr, "NOCLUSTER", cluster_section); + return range_new(rr); + } + + return do_range_expand(rr, res); +} + +static const char** _all_clusters(range_request* rr) +{ + DIR* dir; + struct dirent* dir_entry; + apr_pool_t* pool = range_request_pool(rr); + set* res = set_new(pool, 0); + set* ignore = _get_ignore_set(rr); + char nodes_cf_buf[8192]; + set_element** elts; + const char** table; + int i, n; + + dir = opendir(nodescf_path); + if (!dir) { + range_request_warn(rr, "%s: can't opendir", nodescf_path); + return NULL; + } + + while ( (dir_entry = readdir(dir)) != NULL) { + const char* cluster = dir_entry->d_name; + if (set_get(ignore, cluster) != NULL) continue; + + snprintf(nodes_cf_buf, sizeof nodes_cf_buf, "%s/%s/nodes.cf", + nodescf_path, cluster); + nodes_cf_buf[sizeof nodes_cf_buf - 1] = '\0'; + if (access(nodes_cf_buf, R_OK) == 0) + set_add(res, cluster, 0); + } + + closedir(dir); + + n = res->members; + table = apr_palloc(pool, sizeof(char*) * (n + 1)); + table[n] = NULL; + + elts = set_members(res); + for (i=0; iname; + table[i] = name; + } + + return table; +} + +range* rangefunc_allclusters(range_request* rr, range** r) +{ + range* ret = range_new(rr); + + const char** all_clusters = _all_clusters(rr); + const char** cluster = all_clusters; + int warn_enabled = range_request_warn_enabled(rr); + + if (!cluster) return ret; + + range_request_disable_warns(rr); + while (*cluster) { + range_add(ret, *cluster); + cluster++; + } + if (warn_enabled) range_request_enable_warns(rr); + + return ret; +} + +range* rangefunc_has(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** tag_names = range_get_hostnames(pool, r[0]); + const char** tag_values = range_get_hostnames(pool, r[1]); + + const char* tag_name = tag_names[0]; + const char* tag_value = tag_values[0]; + + const char** all_clusters = _all_clusters(rr); + const char** cluster = all_clusters; + int warn_enabled = range_request_warn_enabled(rr); + + if (!cluster) return ret; + + range_request_disable_warns(rr); + while (*cluster) { + range* vals = _expand_cluster(rr, *cluster, tag_name); + if (set_get(vals->nodes, tag_value) != NULL) { + range_add(ret, *cluster); + } + cluster++; + } + if (warn_enabled) range_request_enable_warns(rr); + + return ret; +} + + +range* rangefunc_mem(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** clusters = range_get_hostnames(pool, r[0]); + const char* cluster = clusters[0]; + const char** wanted = range_get_hostnames(pool, r[1]); + + range* keys = _expand_cluster(rr, cluster, "KEYS"); + const char** all_sections = range_get_hostnames(pool, keys); + const char** p_section = all_sections; + + SECTION: + while (*p_section) { + range* r_s = _expand_cluster(rr, cluster, *p_section); + const char** p_wanted = wanted; + while (*p_wanted) { + if (set_get(r_s->nodes, *p_wanted) != NULL) { + range_add(ret, *p_section++); + goto SECTION; + } + ++p_wanted; + } + ++p_section; + } + + return ret; +} + +range* rangefunc_cluster(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** clusters = range_get_hostnames(pool, r[0]); + const char** p = clusters; + + while (*p) { + const char* colon = strchr(*p, ':'); + range* r1; + if (colon) { + int len = strlen(*p); + int cluster_len = colon - *p; + int section_len = len - cluster_len - 1; + + char* cl = apr_palloc(pool, cluster_len + 1); + char* sec = apr_palloc(pool, section_len + 1); + + memcpy(cl, *p, cluster_len); + cl[cluster_len] = '\0'; + memcpy(sec, colon + 1, section_len); + sec[section_len] = '\0'; + + r1 = _expand_cluster(rr, cl, sec); + } + else + r1 = _expand_cluster(rr, *p, "CLUSTER"); + + if (range_members(r1) > range_members(ret)) { + /* swap them */ + range* tmp = r1; + r1 = ret; + ret = tmp; + } + range_union_inplace(rr, ret, r1); + range_destroy(r1); + ++p; + } + return ret; +} + +range* rangefunc_group(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** groups = range_get_hostnames(pool, r[0]); + + while (*groups) { + if (islower(*groups[0])) + range_union_inplace(rr, ret, _expand_cluster(rr, "HOSTS", *groups)); + else + range_union_inplace(rr, ret, _expand_cluster(rr, "GROUPS", *groups)); + ++groups; + } + return ret; +} + +range* rangefunc_get_admin(range_request* rr, range** r) +{ + range* n = r[0]; + apr_pool_t* pool = range_request_pool(rr); + const char** in_nodes = range_get_hostnames(pool, n); + + range* ret = range_new(rr); + set* node_admin = set_new(pool, 40000); + range* admins_r = _expand_cluster(rr, "HOSTS", "KEYS"); + const char** admins = range_get_hostnames(pool, admins_r); + const char** p_admin = admins; + + while (*p_admin) { + range* nodes_r = _expand_cluster(rr, "HOSTS", *p_admin); + const char** nodes = range_get_hostnames(pool, nodes_r); + const char** p_nodes = nodes; + + while (*p_nodes) { + set_add(node_admin, *p_nodes, (void*)*p_admin); + ++p_nodes; + } + ++p_admin; + } + + while (*in_nodes) { + const char* admin = set_get_data(node_admin, *in_nodes); + if (!admin) { + range_request_warn_type(rr, "NO_ADMIN", *in_nodes); + } + else { + range_add(ret, admin); + } + in_nodes++; + } + + return ret; +} + +static set* _get_clusters(range_request* rr) +{ + const char** all_clusters = _all_clusters(rr); + const char** p_cl = all_clusters; + apr_pool_t* pool = range_request_pool(rr); + set* node_cluster = set_new(pool, 40000); + + if(p_cl == NULL) { + return node_cluster; + } + + while (*p_cl) { + range* nodes_r = _expand_cluster(rr, *p_cl, "ALL"); + const char** nodes = range_get_hostnames(pool, nodes_r); + const char** p_nodes = nodes; + + while (*p_nodes) { + apr_array_header_t* clusters = set_get_data(node_cluster, *p_nodes); + + if (!clusters) { + clusters = apr_array_make(pool, 1, sizeof(char*)); + set_add(node_cluster, *p_nodes, clusters); + } + + *(const char**)apr_array_push(clusters) = *p_cl; + ++p_nodes; + } + ++p_cl; + } + + return node_cluster; +} + +range* rangefunc_get_cluster(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_lr_pool(rr); + const char** nodes = range_get_hostnames(pool, r[0]); + const char** p_nodes = nodes; + set* node_cluster = _get_clusters(rr); + + while (*p_nodes) { + apr_array_header_t* clusters = set_get_data(node_cluster, *p_nodes); + if (!clusters) + range_request_warn_type(rr, "NO_CLUSTER", *p_nodes); + else { + /* just get one */ + const char* cluster = ((const char**)clusters->elts)[0]; + assert(cluster); + range_add(ret, cluster); + } + ++p_nodes; + } + + return ret; +} + +range* rangefunc_clusters(range_request* rr, range** r) +{ + range* ret = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** nodes = range_get_hostnames(pool, r[0]); + const char** p_nodes = nodes; + set* node_cluster = _get_clusters(rr); + + while (*p_nodes) { + apr_array_header_t* clusters = set_get_data(node_cluster, *p_nodes); + if (!clusters) + range_request_warn_type(rr, "NO_CLUSTER", *p_nodes); + else { + /* get all */ + int i; + for (i=0; inelts; ++i) { + const char* cluster = ((const char**)clusters->elts)[i]; + range_add(ret, cluster); + } + } + ++p_nodes; + } + + return ret; +} + +range* rangefunc_get_groups(range_request* rr, range** r) +{ + range* n = r[0]; + apr_pool_t* pool = range_request_pool(rr); + const char** in_nodes = range_get_hostnames(pool, n); + + range* ret = range_new(rr); + set* node_group = set_new(pool, 40000); + range* groups_r = _expand_cluster(rr, "GROUPS", "KEYS"); + const char** groups = range_get_hostnames(pool, groups_r); + const char** p_group = groups; + + while (*p_group) { + range* nodes_r = _expand_cluster(rr, "GROUPS", *p_group); + const char** nodes = range_get_hostnames(pool, nodes_r); + const char** p_nodes = nodes; + + while (*p_nodes) { + apr_array_header_t* my_groups = set_get_data(node_group, *p_nodes); + if (!my_groups) { + my_groups = apr_array_make(pool, 4, sizeof(char*)); + set_add(node_group, *p_nodes, my_groups); + } + *(const char**)apr_array_push(my_groups) = *p_group; + ++p_nodes; + } + ++p_group; + } + + while (*in_nodes) { + apr_array_header_t* my_groups = set_get_data(node_group, *in_nodes); + if (!my_groups) + range_request_warn_type(rr, "NO_GROUPS", *in_nodes); + else { + int i; + for (i=0; inelts; ++i) + range_add(ret, ((const char**)my_groups->elts)[i]); + } + in_nodes++; + } + + return ret; +} diff --git a/libcrange/source/functions/nodescf.h b/libcrange/source/functions/nodescf.h new file mode 100644 index 0000000..e6dadd7 --- /dev/null +++ b/libcrange/source/functions/nodescf.h @@ -0,0 +1,16 @@ +#ifndef NODESCF_H +#define NODESCF_H + +#include "libcrange.h" +#include "set.h" +#include "filecache.h" + +#define GROUP_RE "^([A-Za-z0-9_\\-]+)" +#define INCLUDE_RE "^\\s+INCLUDE\\s+([^#]+)" +#define EXCLUDE_RE "^\\s+EXCLUDE\\s+([^#\\s]+)" +#define DOLLAR_RE "([^\\$]*)\\$(\\w+)" + +struct set *nodescf_read(libcrange *lr, char *filename, char *cname); +int nodescf_replace_dollars(libcrange *lr, filecache *fc, char *clustername); + +#endif diff --git a/libcrange/source/functions/pgsql.c b/libcrange/source/functions/pgsql.c new file mode 100644 index 0000000..e0a23ab --- /dev/null +++ b/libcrange/source/functions/pgsql.c @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include +#include + +#include "set.h" +#include "libcrange.h" +#include "range.h" + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"group", 0}; + return functions; +} + +range* rangefunc_group(range_request* rr, range** r) +{ + range* ret; + const char** members; + int i; + PGconn *conn; + int errors = 0; + + apr_pool_t* pool = range_request_pool(rr); + libcrange* lr = range_request_lr(rr); + + ret = range_new(rr); + members = range_get_hostnames(pool, r[0]); + + const char* default_namespace = libcrange_getcfg(lr, "default_namespace"); + if (!default_namespace) + default_namespace = "yst"; + + if (!(conn = (PGconn *)libcrange_get_cache(lr, "pgsql:conn"))) { + const char* pgsql_user = libcrange_getcfg(lr, "pgsql_user"); + const char* pgsql_db = libcrange_getcfg(lr, "pgsql_db"); + const char* pgsql_passwd = libcrange_getcfg(lr, "pgsql_passwd"); + const char* pgsql_host = libcrange_getcfg(lr, "pgsql_host"); + const char* pgsql_port = libcrange_getcfg(lr, "pgsql_port"); + + if (!pgsql_user) { + range_request_warn(rr, "pgsql no user specified"); + errors++; + } + + if (!pgsql_db) { + range_request_warn(rr, "pgsql no db specified"); + errors++; + } + + if (!pgsql_passwd) { + range_request_warn(rr, "pgsql no passwd specified"); + errors++; + } + + if (!pgsql_host) + pgsql_host = "localhost"; + + if (!pgsql_port) + pgsql_port = "5432"; + + if (errors) + return ret; + + char* conninfo = apr_psprintf + (pool, "host=%s port=%s user=%s password=%s dbname=%s", + pgsql_host, pgsql_port, pgsql_user, pgsql_passwd, + pgsql_db); + + if (!(conn = PQconnectdb(conninfo))) { + range_request_warn(rr, "pgsql dbname=%s: can't connect", + pgsql_db); + return ret; + } + + if (PQstatus(conn) != CONNECTION_OK) { + range_request_warn(rr, "pgsql connection: %s", + PQerrorMessage(conn)); + return ret; + } + + libcrange_set_cache(lr, "pgsql:conn", conn); + } + + for (i = 0; members[i]; i++) { /* for each gemgroup */ + int all = strcmp(members[i], "ALL") == 0; + PGresult* result; + int row, rows; + const char* query; + + if (all) + query = apr_psprintf(pool, "select distinct element " + "from velementgroups where namespace='%s'", + default_namespace); + else + query = apr_psprintf(pool, "select element from velementgroups " + "where namespace='%s' and groupname='%s'", + default_namespace, members[i]); + + result = PQexec(conn, query); + if (!result) { + range_request_warn(rr, "pgsql_group: %s", + PQerrorMessage(conn)); + return ret; + } + + if (( PQresultStatus(result) != PGRES_COMMAND_OK ) && + ( PQresultStatus(result) != PGRES_TUPLES_OK )) + { + range_request_warn(rr, "pgsql_group: %s", + PQerrorMessage(conn)); + return ret; + } + + rows = PQntuples(result); + for (row=0; row < rows; ++row) { + const char* element = PQgetvalue(result, row, 0); + range_add(ret, element); + } + + PQclear(result); + } + + return ret; +} diff --git a/libcrange/source/functions/tinydns_ip.c b/libcrange/source/functions/tinydns_ip.c new file mode 100644 index 0000000..ad2d881 --- /dev/null +++ b/libcrange/source/functions/tinydns_ip.c @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "range_request.h" +#include "set.h" +#include "libcrange.h" +#include "tinydns_ip.h" + +typedef struct cache_entry +{ + time_t mtime; + apr_pool_t* pool; + set* hosts_ip; + set* cnames; +} cache_entry; + +static cache_entry* _dummy_cache_entry(apr_pool_t* pool) +{ + cache_entry* e = apr_palloc(pool, sizeof(cache_entry)); + e->mtime = 0; + e->cnames = e->hosts_ip = set_new(pool, 0); + return e; +} + +static cache_entry* tinydns_read(libcrange* lr) +{ + const char* error; + int offset; + static pcre* a_re = NULL; + static pcre* cname_re = NULL; + + apr_pool_t* pool = libcrange_get_pool(lr); + cache_entry* e; + struct stat st; + char line[8192]; + FILE* fp; + const char* dns_file = libcrange_getcfg(lr, "dns_data_file"); + if (!dns_file) dns_file = DNS_FILE; + + if (stat(dns_file, &st) < 0) { + fprintf(stderr, "Can't stat %s", dns_file); + /* dummy cache */ + return _dummy_cache_entry(pool); + } + + e = libcrange_get_cache(lr, "dns:tinydns_data"); + if (e && e->mtime == st.st_mtime) + return e; + + if (!e) { + e = apr_palloc(pool, sizeof(cache_entry)); + apr_pool_create(&e->pool, pool); + libcrange_set_cache(lr, "dns:tinydns_data", e); + } + else + apr_pool_clear(e->pool); + + e->mtime = st.st_mtime; + e->hosts_ip = set_new(e->pool, 50000); + e->cnames = set_new(e->pool, 1000); + + if (!a_re) { + a_re = pcre_compile(A_RE, 0, &error, &offset, NULL); + cname_re = pcre_compile(CNAME_RE, 0, &error, &offset, NULL); + } + + fp = fopen(dns_file, "r"); + if (!fp) { + fprintf(stderr, "Can't open %s: %s", dns_file, strerror(errno)); + return _dummy_cache_entry(pool);; + } + + while (fgets(line, sizeof line, fp)) { + int ovector[30]; + int count; + int len; + + line[sizeof line - 1] = '\0'; + len = strlen(line); + if (len+1 >= sizeof(line) && line[len - 1] != '\n') { + /* incomplete line */ + fprintf(stderr, "%s: lines > %d chars not supported\n", dns_file, + sizeof line); + exit(-1); + } + + count = pcre_exec(a_re, NULL, line, len, + 0, 0, ovector, 30); + if (count > 0) { + char* host = &line[ovector[2]]; + char* ip = &line[ovector[4]]; + line[ovector[3]] = line[ovector[5]] = '\0'; + + set_add(e->hosts_ip, host, ip_new(e->pool, ip)); + } else { + count = pcre_exec(cname_re, NULL, line, len, + 0, 0, ovector, 30); + if (count > 0) { + char* alias = &line[ovector[2]]; + char* canon = &line[ovector[4]]; + line[ovector[3]] = line[ovector[5]] = '\0'; + + set_add(e->cnames, alias, apr_pstrdup(e->pool, canon)); + } + } + } + + return e; +} + +ip* tinydns_get_ip(range_request* rr, const char* hostname) +{ + struct hostent* h; + apr_pool_t* pool = range_request_pool(rr); + + if (isdigit(hostname[0])) + return ip_new(pool, hostname); + + h = gethostbyname(hostname); + if (!h) + return NULL; + + if (h->h_addrtype != AF_INET || h->h_length != 4) + /* no IPv4 addresses */ + return NULL; + + return ip_new(pool, apr_psprintf(pool, + "%u.%u.%u.%u", + ((unsigned char*)h->h_addr)[0], + ((unsigned char*)h->h_addr)[1], + ((unsigned char*)h->h_addr)[2], + ((unsigned char*)h->h_addr)[3])); +} + + +ip_host** tinydns_all_ip_hosts(libcrange* lr, apr_pool_t* pool) +{ + ip_host** result; + ip_host** p; + set_element** hosts; + cache_entry* e = tinydns_read(lr); + + p = result = apr_palloc(pool, sizeof(ip_host*) * (e->hosts_ip->members + 1)); + for (hosts = set_members(e->hosts_ip); *hosts; ++hosts) { + const char* host = (*hosts)->name; + const ip* host_ip = (*hosts)->data; + ip_host* iph = apr_palloc(pool, sizeof(ip_host)); + iph->hostname = host; + iph->ip = host_ip; + *p++ = iph; + } + *p = NULL; + + return result; +} + +ip* ip_new(apr_pool_t* pool, const char* ipaddr) +{ + ip* i; + + i = apr_palloc(pool, sizeof(ip)); + i->binary = str2ip(ipaddr); + i->str = apr_pstrdup(pool, ipaddr); + + return i; +} + +unsigned str2ip(const char* str) +{ + unsigned a, b, c, d; + if (sscanf(str, "%u.%u.%u.%u", &a, &b, &c, &d) == 4) + return (a<<24) + (b<<16) + (c<<8) + d; + + return 0; +} diff --git a/libcrange/source/functions/tinydns_ip.h b/libcrange/source/functions/tinydns_ip.h new file mode 100644 index 0000000..d27c5f8 --- /dev/null +++ b/libcrange/source/functions/tinydns_ip.h @@ -0,0 +1,28 @@ +#ifndef TINYDNS_IP_H +#define TINYDNS_IP_H + +#include "libcrange.h" + +#define A_RE "^\\+([^:]+):([^:]+):0" +#define CNAME_RE "^C([^:]+):([^:]+)\\.:0" +#define DNS_FILE "/export/crawlspace/tinydns-data/root/data" + +typedef struct ip +{ + unsigned binary; + const char* str; +} ip; + +typedef struct ip_host +{ + const ip* ip; + const char* hostname; +} ip_host; + +unsigned str2ip(const char* str); +ip* ip_new(apr_pool_t* pool, const char* ipaddr); +ip* tinydns_get_ip(range_request* rr, const char* hostname); +ip_host** tinydns_all_ip_hosts(libcrange* lr, apr_pool_t* pool); + + +#endif diff --git a/libcrange/source/functions/yst-ip-list.c b/libcrange/source/functions/yst-ip-list.c new file mode 100644 index 0000000..8b21a5d --- /dev/null +++ b/libcrange/source/functions/yst-ip-list.c @@ -0,0 +1,104 @@ +#include +#include + +#include "set.h" +#include "libcrange.h" +#include "range.h" +#include "tinydns_ip.h" +#include "hosts-netblocks.h" + +const char** functions_provided(libcrange* lr) +{ + static const char* functions[] = {"vlan", "dc", "hosts_v", "hosts_dc", "vlans_dc", 0}; + return functions; +} + +range* rangefunc_vlans_dc(range_request* rr, range** r) +{ + range* result = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** members = range_get_hostnames(pool, r[0]); + int i; + + for (i=0; members[i]; ++i) { + const char* dc = members[i]; + range_union_inplace(rr, result, netblocks_for_dc(rr, dc)); + } + return result; +} + +range* rangefunc_hosts_dc(range_request* rr, range** r) +{ + int i; + range* result = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** members = range_get_hostnames(pool, r[0]); + + for (i = 0; members[i]; ++i) { + const char* dc = members[i]; + range_union_inplace(rr, result, hosts_in_dc(rr, dc)); + } + return result; +} + +range* rangefunc_hosts_v(range_request* rr, range** r) +{ + int i; + range* result = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + const char** members = range_get_hostnames(pool, r[0]); + + for (i = 0; members[i]; ++i) { + char net_key[32]; + const char* net = members[i]; + const netblock* blk = netblock_from_string(pool, net); + netblock_key(blk, net_key, sizeof net_key); + range_union_inplace(rr, result, hosts_in_netblock(rr, net_key)); + } + return result; +} + +range* rangefunc_vlan(range_request* rr, range** r) +{ + range* ret; + const char** members; + apr_pool_t* pool = range_request_pool(rr); + int i; + + members = range_get_hostnames(pool, r[0]); + ret = range_new(rr); + ret->quoted = 1; + + for (i = 0; members[i]; ++i) { /* for each node */ + const char* node = members[i]; + const netblock* block = netblock_for_host(rr, node); + if (block) + range_add(ret, block->str); + else + range_request_warn_type(rr, "HOST_NO_NETBLOCK", node); + } + + return ret; +} + +range* rangefunc_dc(range_request* rr, range **r) +{ + range* ret; + const char** members; + apr_pool_t* pool = range_request_pool(rr); + int i; + + members = range_get_hostnames(pool, r[0]); + ret = range_new(rr); + + for (i = 0; members[i]; ++i) { /* for each node */ + const char* node = members[i]; + const char* dc = dc_for_host(rr, node); + if (dc) + range_add(ret, dc); + else + range_request_warn_type(rr, "NO_COLO", node); + } + + return ret; +} diff --git a/libcrange/source/m4/Makefile.am b/libcrange/source/m4/Makefile.am new file mode 100644 index 0000000..e69de29 diff --git a/libcrange/source/perl/Libcrange.xs b/libcrange/source/perl/Libcrange.xs new file mode 100644 index 0000000..0cd3b51 --- /dev/null +++ b/libcrange/source/perl/Libcrange.xs @@ -0,0 +1,74 @@ +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" + + +#include + +/* + +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +*/ + +static void expand(long ptr, const char* range) +{ + dXSARGS; + range_request* rr = (range_request*) ptr; + const char** nodes; + + sp = mark; + rr = range_expand_rr(rr, range); + nodes = range_request_nodes(rr); + while (*nodes) + XPUSHs(sv_2mortal(newSVpv(*nodes++, 0))); + + PUTBACK; +} + +#define LR(rr) (range_request_lr((range_request*)rr)) +#define RR(rr) ((range_request*)rr) + +MODULE = Libcrange PACKAGE = Libcrange + +PROTOTYPES: DISABLE + +void +expand(rr, range) + long rr + const char* range + PREINIT: + I32* temp; + PPCODE: + temp = PL_markstack_ptr++; + expand(rr, range); + if (PL_markstack_ptr != temp) { + PL_markstack_ptr = temp; + XSRETURN_EMPTY; + } + return; + +const char* +get_var(rr, var) + long rr + const char* var + CODE: + RETVAL = libcrange_getcfg(LR(rr), var); + OUTPUT: + RETVAL + +void +warn_type(rr, type, msg) + long rr + const char* type + const char* msg + CODE: + range_request_warn_type(RR(rr), type, msg); + +void +warn(rr, msg) + long rr + const char* msg + CODE: + range_request_warn(RR(rr), "%s", msg); diff --git a/libcrange/source/perl/Makefile.PL.in b/libcrange/source/perl/Makefile.PL.in new file mode 100644 index 0000000..2376a7d --- /dev/null +++ b/libcrange/source/perl/Makefile.PL.in @@ -0,0 +1,25 @@ +use 5.006; +use ExtUtils::MakeMaker; + +my $ld_flags = "@APR_LIBS_LD@"; +my $c_flags = "@APR_CFLAGS@"; + +WriteMakefile( + NAME => 'Libcrange', + VERSION_FROM => 'lib/Libcrange.pm', # finds $VERSION + PREREQ_PM => {}, # e.g., Module::Name => 1.1 + ( + $] >= 5.005 + ? ## Add these new keywords supported since 5.005 + ( + ABSTRACT_FROM => 'lib/Libcrange.pm', # retrieve abstract from module + AUTHOR => 'Daniel Muino ' + ) + : () + ), + LIBS => ["-L/usr/local/lib -lcrange $ld_flags"], # e.g., '-lm' + DEFINE => '', # e.g., '-DHAVE_SOMETHING' + INC => "-I. -I../src $c_flags", # e.g., '-I. -I/usr/include/other' + # Un-comment this if you add C files to link with later: + # OBJECT => '$(O_FILES)', # link all the C files too +); diff --git a/libcrange/source/perl/build.in b/libcrange/source/perl/build.in new file mode 100644 index 0000000..0071875 --- /dev/null +++ b/libcrange/source/perl/build.in @@ -0,0 +1,8 @@ +@PERL@ Makefile.PL +make +make install DESTDIR=$DESTDIR +find $DESTDIR -name perllocal.pod -type f -exec rm \{\} \; +#if [ -f /usr/lib/rpm/redhat/brp-compress ]; then +# RPM_BUILD_ROOT=$DESTDIR /usr/lib/rpm/redhat/brp-compress +#fi + diff --git a/libcrange/source/perl/lib/Libcrange.pm b/libcrange/source/perl/lib/Libcrange.pm new file mode 100644 index 0000000..98daf5e --- /dev/null +++ b/libcrange/source/perl/lib/Libcrange.pm @@ -0,0 +1,86 @@ +package Libcrange; + +use 5.006; +use strict; +use warnings; + +require Exporter; + +our @ISA = qw(Exporter); + +# Items to export into callers namespace by default. Note: do not export +# names by default without a very good reason. Use EXPORT_OK instead. +# Do not simply export all your public functions/methods/constants. + +# This allows declaration use Libcrange ':all'; +# If you do not need this, moving things directly into @EXPORT or @EXPORT_OK +# will save memory. +our %EXPORT_TAGS = ( 'all' => [ qw( + +) ] ); + +our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); + +our @EXPORT = qw( + +); + +our $VERSION = '0.01'; + +require XSLoader; +XSLoader::load('Libcrange', $VERSION); + +# Preloaded methods go here. + +1; +__END__ +# Below is stub documentation for your module. You'd better edit it! + +=head1 NAME + +Libcrange - Perl extension for blah blah blah + +=head1 SYNOPSIS + + use Libcrange; + blah blah blah + +=head1 DESCRIPTION + +Stub documentation for Libcrange, created by h2xs. It looks like the +author of the extension was negligent enough to leave the stub +unedited. + +Blah blah blah. + +=head2 EXPORT + +None by default. + + + +=head1 SEE ALSO + +Mention other useful documentation such as the documentation of +related modules or operating system documentation (such as man pages +in UNIX), or any relevant external documentation such as RFCs or +standards. + +If you have a mailing list set up for your module, mention it here. + +If you have a web site set up for your module, mention it here. + +=head1 AUTHOR + +Daniel Muino, Edmuino@localdomainE + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2006 by Yahoo! Inc. + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself, either Perl version 5.8.8 or, +at your option, any later version of Perl 5 you may have available. + + +=cut diff --git a/libcrange/source/src/Makefile.am b/libcrange/source/src/Makefile.am new file mode 100644 index 0000000..e1fa915 --- /dev/null +++ b/libcrange/source/src/Makefile.am @@ -0,0 +1,23 @@ +AM_YFLAGS = -d +AM_CFLAGS = -fPIC -Wall + +bin_PROGRAMS = crange +crange_SOURCES = main.c +crange_CFLAGS = @APR_CFLAGS@ +crange_LDFLAGS = -lcrange @PERL_LIBS@ @APR_LIBS@ +include_HEADERS = libcrange.h + +lib_LTLIBRARIES = libcrange.la +libcrange_la_SOURCES = range_parser.y range_scanner.l \ + set.c range_request.c \ + range_sort.c range_parts.c perl_functions.c \ + libcrange.c ast.c range_compress.c \ + range.c + +libcrange_la_CFLAGS = -Wall -DLIBCRANGE_FUNCDIR=\"$(pkglibdir)\" @PERL_CFLAGS@ @PCRE_CFLAGS@ @APR_CFLAGS@ +libcrange_la_LDFLAGS = @PERL_LIBS@ @PCRE_LIBS@ @APR_LIBS@ + + + + + diff --git a/libcrange/source/src/ast.c b/libcrange/source/src/ast.c new file mode 100644 index 0000000..25484bf --- /dev/null +++ b/libcrange/source/src/ast.c @@ -0,0 +1,120 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include "ast.h" +#include "range.h" +#include "libcrange.h" + +rangeast* range_ast_new(apr_pool_t* pool, rangetype type) +{ + rangeast* r = (rangeast *)apr_palloc(pool, sizeof(rangeast)); + r->children = NULL; + r->next = NULL; + r->type = type; + + return r; +} + +range* range_evaluate(range_request* rr, const rangeast* ast) +{ + range* r; + int i; + rangeast* rtmp; + range** ranges; + range* r1; + range* r2; + range* r3; + apr_pool_t* pool = range_request_pool(rr); + + switch (ast->type) { + case AST_LITERAL: + r = range_from_literal(rr, ast->data.string); + return r; + case AST_NONRANGE_LITERAL: + r = range_from_nonrange_literal(rr, ast->data.string); + return r; + case AST_UNION: + r1 = range_evaluate(rr, ast->children); + r2 = range_evaluate(rr, ast->children->next); + if (range_members(r1) > range_members(r2)) { + range_union_inplace(rr, r1, r2); + range_destroy(r2); + return r1; + } else { + range_union_inplace(rr, r2, r1); + range_destroy(r1); + return r2; + } + case AST_GROUP: + r1 = range_evaluate(rr, ast->children); + r = range_from_group(rr, r1); + range_destroy(r1); + return r; + case AST_DIFF: + r1 = range_evaluate(rr, ast->children); + if (ast->children->next->type == AST_REGEX) + r2 = range_from_match(rr, r1, ast->children->next->data.string); + else + r2 = range_evaluate(rr, ast->children->next); + range_diff_inplace(rr, r1, r2); + return r1; + case AST_INTER: + r1 = range_evaluate(rr, ast->children); + if (ast->children->next->type == AST_REGEX) + r2 = range_from_match(rr, r1, ast->children->next->data.string); + else + r2 = range_evaluate(rr, ast->children->next); + r = range_from_inter(rr, r1, r2); + range_destroy(r1); + range_destroy(r2); + return r; + case AST_NOT: + r1 = range_from_group(rr, range_from_literal(rr, "ALL")); + r2 = range_evaluate(rr, ast->children); + range_diff_inplace(rr, r1, r2); + range_destroy(r2); + return r1; + case AST_REGEX: + r1 = range_from_group(rr, range_from_literal(rr, "ALL")); + r = range_from_match(rr, r1, ast->data.string); + range_destroy(r1); + return r; + case AST_PARTS: + r = range_from_rangeparts(rr, ast->data.parts); + return r; + case AST_BRACES: + r1 = range_evaluate(rr, ast->children); + r2 = range_evaluate(rr, ast->children->next); + r3 = range_evaluate(rr, ast->children->next->next); + r = range_from_braces(rr, r1, r2, r3); + range_destroy(r1); + range_destroy(r2); + range_destroy(r3); + return r; + case AST_FUNCTION: + i=0; + for (rtmp = ast->children; rtmp; rtmp = rtmp->next) + i++; + + ranges = (range **)apr_palloc(pool, sizeof(range *) * (i+1)); + ranges[i] = NULL; + i=0; + for (rtmp = ast->children; rtmp; rtmp = rtmp->next) + ranges[i++] = range_evaluate(rr, rtmp); + + r = range_from_function(rr, ast->data.string, (const range**)ranges); + for (i=0; ranges[i]; i++) range_destroy(ranges[i]); + + return r; + case AST_NOTHING: + r = range_new(rr); + return r; + default: + fprintf(stderr, "ERROR IN LIBCRANGE: Corrupted AST\n"); + abort(); + } +} diff --git a/libcrange/source/src/ast.h b/libcrange/source/src/ast.h new file mode 100644 index 0000000..993a0f7 --- /dev/null +++ b/libcrange/source/src/ast.h @@ -0,0 +1,43 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef AST_H +#define AST_H + +#include "libcrange.h" +#include "range.h" + +typedef enum { + AST_LITERAL, + AST_NONRANGE_LITERAL, + AST_UNION, + AST_GROUP, + AST_BRACES, + AST_DIFF, + AST_INTER, + AST_NOT, + AST_REGEX, + AST_PARTS, + AST_FUNCTION, + AST_NOTHING +} rangetype; + +typedef struct rangeast +{ + rangetype type; + union + { + char *string; + rangeparts *parts; + } data; + + struct rangeast *children; + struct rangeast *next; +} rangeast; + +rangeast* range_ast_new(apr_pool_t* pool, rangetype type); +range* range_evaluate(range_request* rr, const rangeast* ast); + +#endif diff --git a/libcrange/source/src/config.h b/libcrange/source/src/config.h new file mode 100644 index 0000000..9c29222 --- /dev/null +++ b/libcrange/source/src/config.h @@ -0,0 +1,104 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +/* config.h. Generated by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#define HAVE_DECL_STRERROR_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `crypt' library (-lcrypt). */ +#define HAVE_LIBCRYPT 1 + +/* Define to 1 if you have the `m' library (-lm). */ +#define HAVE_LIBM 1 + +/* Define to 1 if you have the `mysqlclient' library (-lmysqlclient). */ +/* #undef HAVE_LIBMYSQLCLIENT */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +#define HAVE_LIBNSL 1 + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#define HAVE_LIBPTHREAD 1 + +/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */ +#define HAVE_LIBSQLITE3 1 + +/* Define to 1 if you have the `z' library (-lz). */ +#define HAVE_LIBZ 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if `stat' has the bug that it succeeds when given the + zero-length file name argument. */ +/* #undef HAVE_STAT_EMPTY_STRING_BUG */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if `lstat' dereferences a symlink specified with a trailing + slash. */ +#define LSTAT_FOLLOWS_SLASHED_SYMLINK 1 + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "BUG-REPORT-ADDRESS" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "FULL-PACKAGE-NAME" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "FULL-PACKAGE-NAME VERSION" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "full-package-name" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "VERSION" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if strerror_r returns char *. */ +#define STRERROR_R_CHAR_P 1 + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +#define YYTEXT_POINTER 1 + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ diff --git a/libcrange/source/src/crange.i b/libcrange/source/src/crange.i new file mode 100644 index 0000000..e331ba0 --- /dev/null +++ b/libcrange/source/src/crange.i @@ -0,0 +1,25 @@ +%module Crange +%{ +#include "libcrange.h" +%} +%include "libcrange.h" +%inline %{ +char **string_array(int size) { + int i; + char **a = (char **)malloc(sizeof(char *) * size); + for(i = 0; i < size; i++) { + a[i] = NULL; + } + return a; +} +void string_destroy(char **a) { + free(a); +} +void string_set(char **a, int i, char *val) { + a[i] = val; +} +char *string_get(char **a, int i) { + return a[i]; +} + +%} diff --git a/libcrange/source/src/libcrange-embed.pl b/libcrange/source/src/libcrange-embed.pl new file mode 100644 index 0000000..c11540c --- /dev/null +++ b/libcrange/source/src/libcrange-embed.pl @@ -0,0 +1,33 @@ +package Seco::Embed::Range; + +use lib '/var/libcrange/perl'; +use Data::Dumper; + +my %functions; + +sub load_file { + my ($prefix, $module) = @_; + +# print STDERR "Loading $module prefix=[$prefix]\n"; + require "$module.pm"; + my @functions = $module->functions_provided; + my @mapped_functions = map { "$prefix$_" } @functions; + + for (@functions) { + $functions{"$prefix$_"} = \&{"${module}::$_"}; + } + + @mapped_functions; +} + +sub call_func { + my $func = shift; + my @args = @_; + + my $fun_ref = $functions{$func}; + die "No function $func\n" unless $fun_ref; + + &$fun_ref(@args); +} + +1; diff --git a/libcrange/source/src/libcrange.c b/libcrange/source/src/libcrange.c new file mode 100644 index 0000000..57cfb02 --- /dev/null +++ b/libcrange/source/src/libcrange.c @@ -0,0 +1,446 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libcrange.h" +#include "range.h" +#include "range_compress.h" +#include "perl_functions.h" +#include "range_request.h" + +libcrange* static_lr = NULL; +static int initd = 0; + +struct libcrange +{ + set* caches; + set* functions; + set* perl_functions; + set* vars; + + apr_pool_t* pool; + const char* default_domain; + const char* confdir; + const char* config_file; + const char* funcdir; + int want_caching; +}; + +static int parse_config_file(libcrange* lr); +libcrange* libcrange_new(apr_pool_t* pool, const char* config_file) +{ + libcrange* lr; + + if (!initd) { + initd = 1; + apr_initialize(); + atexit(apr_terminate); + } + + lr = apr_palloc(pool, sizeof(libcrange)); + lr->pool = pool; + lr->caches = set_new(pool, 0); + lr->default_domain = "inktomisearch.com"; + lr->funcdir = LIBCRANGE_FUNCDIR; + lr->want_caching = 1; + lr->config_file = config_file ? config_file : LIBCRANGE_CONF; + lr->functions = set_new(pool, 0); + lr->perl_functions = NULL; + lr->vars = set_new(pool, 0); + + if (access(lr->config_file, R_OK) != 0) + return lr; /* no config file, don't load any modules */ + + if (parse_config_file(lr) < 0) + return NULL; + + return lr; +} + +char* libcrange_get_pcre_substring(apr_pool_t* pool, const char* string, + int offsets[], int substr) +{ + int idx = substr * 2; + char* new_str; + int size = offsets[idx+1] - offsets[idx]; + + if (size == 0) + return ""; + + new_str = apr_palloc(pool, size + 1); + memcpy(new_str, &string[offsets[idx]], size); + new_str[size] = '\0'; + return new_str; +} + +apr_pool_t* libcrange_get_pool(libcrange* lr) +{ + return lr->pool; +} + +const char* libcrange_get_default_domain(libcrange* lr) +{ + assert(lr); + return lr->default_domain; +} + +static apr_pool_t* static_pool = NULL; +static void destroy_static_pool(void) +{ + apr_pool_destroy(static_pool); +} + +libcrange* get_static_lr(void) +{ + if (static_lr == NULL) { + apr_pool_create(&static_pool, NULL); + static_lr = libcrange_new(static_pool, NULL); + atexit(destroy_static_pool); + } + + return static_lr; +} + +void libcrange_set_default_domain(libcrange* lr, const char* domain) +{ + lr->default_domain = apr_pstrdup(lr->pool, domain); +} + +const char* libcrange_get_perl_module(libcrange* lr, const char* funcname) +{ + assert(lr); + if (lr->perl_functions) + return set_get_data(lr->perl_functions, funcname); + else + return NULL; +} + +void* libcrange_get_function(libcrange* lr, const char* funcname) +{ + set* functions; + assert(lr); + assert(funcname); + + functions = lr->functions; + return set_get_data(functions, funcname); +} + + +void* libcrange_get_cache(libcrange* lr, const char* name) +{ + set_element* se; + if (lr == NULL) lr = get_static_lr(); + + if ((se = set_get(lr->caches, name))) + return se->data; + else + return NULL; +} + +void libcrange_clear_caches(libcrange* lr) +{ + if (lr == NULL) lr = get_static_lr(); + lr->caches = set_new(lr->pool, 317); +} + +void libcrange_set_cache(libcrange* lr, const char* name, void* data) +{ + if (lr == NULL) lr = get_static_lr(); + if (lr->want_caching) + set_add(lr->caches, name, data); +} + +const char* range_compress(libcrange* lr, apr_pool_t* p, const char** nodes) +{ + range* r; + range_request* rr; + + if (lr == NULL) lr = get_static_lr(); + assert(lr); + + rr = range_request_new(lr, p); + + r = range_from_hostnames(rr, nodes); + return do_range_compress(rr, r); +} + +range_request* range_expand(libcrange* lr, apr_pool_t* pool, const char* text) +{ + range_request* rr; + + if (lr == NULL) lr = get_static_lr(); + assert(lr); + + rr = range_request_new(lr, pool); + do_range_expand(rr, text); + + return rr; +} + +range_request* range_expand_rr(range_request* rr, const char* text) +{ + assert(rr); + do_range_expand(rr, text); + return rr; +} + +void libcrange_want_caching(libcrange* lr, int want) +{ + if (lr == NULL) lr = get_static_lr(); + lr->want_caching = want; +} + +const char* libcrange_get_version(void) +{ + return LIBCRANGE_VERSION; +} + +const char* libcrange_getcfg(libcrange* lr, const char* what) +{ + if (lr == NULL) lr = get_static_lr(); + + return set_get_data(lr->vars, what); +} + +static const char** get_function_names(libcrange* lr, void *handle, + const char* module) +{ + const char** (*f)(libcrange*); + const char* err; + + *(void **)(&f) = dlsym(handle, "functions_provided"); + if ((err = dlerror()) != NULL) { + fprintf(stderr, "Module %s: error getting functions_provided()\n", + module); + return NULL; + } + + return (*f)(lr); +} + +static int add_function(libcrange* lr, set* functions, void* handle, + const char* module, const char* prefix, + const char* function) +{ + void *f; /* it's actually a function pointer but since we're adding it + * to a set, let's leave it as void * */ + const char* err; + char function_name[512] = "rangefunc_"; + strncat(function_name, function, sizeof function_name); + function_name[sizeof function_name - 1] = '\0'; + + f = dlsym(handle, function_name); + if ((err = dlerror()) != NULL) { + fprintf(stderr, "Module %s: error getting %s\n", + module, function_name); + return 1; + } + + /* reusing function_name */ + assert(strlen(prefix) < 16); + assert(strlen(function) < 256); + + strcpy(function_name, prefix); + strcat(function_name, function); + set_add(functions, function_name, f); + return 0; +} + +static int add_functions_from_module(libcrange* lr, set* functions, + const char* module, const char* prefix) +{ + void* handle; + char filename[512]; + const char** all_functions; + + snprintf(filename, sizeof filename, "%s/%s.so", lr->funcdir, module); + filename[sizeof filename - 1] = '\0'; + + if (access(filename, R_OK) != 0) { + fprintf(stderr, "module %s (can't read %s.)\n", + module, filename); + return 1; + } + + if ((handle = dlopen(filename, RTLD_NOW)) == NULL) { + fprintf(stderr, "%s: can't dlopen: %s\n", filename, dlerror()); + return 1; + } + + dlerror(); /* Clear any existing errors */ + + all_functions = get_function_names(lr, handle, module); + if (all_functions == NULL) + return 1; + + while (*all_functions) { + int err = add_function(lr, functions, handle, + module, prefix, *all_functions++); + if (err != 0) + return err; + } + + return 0; +} + +#define LOADMODULE_RE "^\\s*loadmodule\\s+([-\\S]+)(?:\\s+prefix=([-\\w]+))?\\s*$" +#define PERLMODULE_RE "^\\s*perlmodule\\s+([-\\S]+)(?:\\s+prefix=([-\\w]+))?\\s*$" +#define VAR_RE "^\\s*([-\\w]+)\\s*=\\s*(\\S+)\\s*$" + +static int parse_config_file(libcrange* lr) +{ + int ovector[30]; + const char* error; + int err_offset; + int line_no = 0; + char line[256]; + pcre* loadmodule_re; + pcre* perlmodule_re; + pcre* var_re; + FILE* fp; + const char* config_file; + + set* functions; + set* perl_functions = NULL; + set* vars; + + assert(lr); + assert(lr->config_file); + + config_file = lr->config_file; + + if (config_file == NULL) return 0; + fp = fopen(config_file, "r"); + if (!fp) { + fprintf(stderr, "%s: (%d) %s\n", + config_file, errno, strerror(errno)); + return -1; + } + + functions = lr->functions; + vars = lr->vars; + + /* compile the regex */ + loadmodule_re = pcre_compile(LOADMODULE_RE, 0, &error, + &err_offset, NULL); + var_re = pcre_compile(VAR_RE, 0, &error, + &err_offset, NULL); + + perlmodule_re = pcre_compile(PERLMODULE_RE, 0, &error, + &err_offset, NULL); + assert(loadmodule_re); + assert(var_re); + assert(perlmodule_re); + + while (fgets(line, sizeof line, fp)) { + int err; + int count; + int n = strlen(line); + const char* module; + const char* prefix; + const char* var; + const char* value; + + ++line_no; + if (line[n - 1] != '\n') { + fprintf(stderr, "%s:%d line too long - aborting\n", + lr->config_file, line_no); + fclose(fp); + return -1; + } + if (line[0] == '#') continue; + + line[--n] = '\0'; /* chop */ + if (!n) continue; + + /* if it's a loadmodule line */ + count = pcre_exec(loadmodule_re, NULL, line, n, + 0, 0, ovector, 30); + if (count > 1) { + module = &line[ovector[2]]; + line[ovector[3]] = '\0'; + if (count > 2) { + prefix = &line[ovector[4]]; + line[ovector[5]] = '\0'; + } + else + prefix = ""; + + err = add_functions_from_module(lr, functions, module, prefix); + + if (err) { + fclose(fp); + return -1; + } + + continue; + } + + /* var = expression */ + count = pcre_exec(var_re, NULL, line, n, + 0, 0, ovector, 30); + if (count > 1) { + var = &line[ovector[2]]; + line[ovector[3]] = '\0'; + + value = &line[ovector[4]]; + line[ovector[5]] = '\0'; + + set_add(vars, var, apr_pstrdup(lr->pool, value)); + + continue; + } + + /* perlmodule */ + count = pcre_exec(perlmodule_re, NULL, line, n, + 0, 0, ovector, 30); + if (count > 1) { + module = &line[ovector[2]]; + line[ovector[3]] = '\0'; + if (count > 2) { + prefix = &line[ovector[4]]; + line[ovector[5]] = '\0'; + } + else + prefix = ""; + + if (!perl_functions) + perl_functions = set_new(lr->pool, 0); + + err = add_functions_from_perlmodule(lr, lr->pool, + perl_functions, module, prefix); + + if (err) { + fclose(fp); + return -1; + } + + continue; + } + + /* don't know how to parse: not a loadmoule or var=value */ + fprintf(stderr, "%s:%d syntax error [%s]\n", + lr->config_file, line_no, line); + fclose(fp); + return -1; + } + + fclose(fp); + + lr->perl_functions = perl_functions; + lr->functions = functions; + lr->vars = vars; + return 0; +} diff --git a/libcrange/source/src/libcrange.h b/libcrange/source/src/libcrange.h new file mode 100644 index 0000000..4e38da6 --- /dev/null +++ b/libcrange/source/src/libcrange.h @@ -0,0 +1,76 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef LIBCRANGE_H +#define LIBCRANGE_H + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LIBCRANGE_VERSION "1.0" + +#define LIBCRANGE_CONF "/etc/libcrange.conf" +#define DEFAULT_SQLITE_DB "/var/libcrange/secodata.dat" +#define LIBCRANGE_FUNCDIR "/usr/lib/libcrange" + +struct libcrange; +typedef struct libcrange libcrange; + +struct range_request; +typedef struct range_request range_request; + +/* These are the commonly used functions - pass lr == NULL unless + * you're doing something special */ +const char* range_compress(libcrange* lr, apr_pool_t* pool, + const char** nodes); + +struct range_request* range_expand(libcrange* lr, apr_pool_t* pool, + const char* text); + + +struct range_request* range_expand_rr(range_request* rr, const char* text); + +/* return a compressed version of this range_request results */ +const char* range_request_compressed(struct range_request* rr); + +/* the warnings for this range request */ +const char* range_request_warnings(struct range_request* rr); + +/* the result as a NULL terminated array of strings */ +const char** range_request_nodes(struct range_request* rr); + +/* did we generate warnings */ +int range_request_has_warnings(struct range_request* rr); + +/* get the current library version */ +const char* libcrange_get_version(void); + +/* get the libcrange* from this range_request */ +struct libcrange* range_request_lr(range_request* rr); + +/* these functions are mostly used by the modules */ +libcrange* libcrange_new(apr_pool_t* pool, const char* config_file); +apr_pool_t* libcrange_get_pool(libcrange* lr); +void libcrange_set_cache(libcrange* lr, const char *name, void *data); +void* libcrange_get_cache(libcrange* lr, const char *name); +void libcrange_clear_caches(libcrange* lr); +void libcrange_want_caching(libcrange* lr, int want); +const char* libcrange_getcfg(libcrange* lr, const char* what); +void libcrange_set_default_domain(libcrange* lr, const char* domain); +const char* libcrange_get_perl_module(libcrange* lr, const char* funcname); +const char* libcrange_get_default_domain(libcrange* lr); +void* libcrange_get_function(libcrange* lr, const char* funcname); +char* libcrange_get_pcre_substring(apr_pool_t* pool, const char* string, + int offsets[], int substr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/libcrange/source/src/main.c b/libcrange/source/src/main.c new file mode 100644 index 0000000..c794b26 --- /dev/null +++ b/libcrange/source/src/main.c @@ -0,0 +1,28 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include "libcrange.h" +#include + +int main(int argc, char const* const* argv) +{ + apr_pool_t* pool; + struct range_request* rr; + + apr_app_initialize(&argc, &argv, NULL); + atexit(apr_terminate); + apr_pool_create(&pool, NULL); + + rr = range_expand(NULL, pool, argv[1]); + printf("%s\n", range_request_compressed(rr)); + + if (range_request_has_warnings(rr)) + printf("%s\n", range_request_warnings(rr)); + + apr_pool_destroy(pool); + return 0; +} diff --git a/libcrange/source/src/perl_functions.c b/libcrange/source/src/perl_functions.c new file mode 100644 index 0000000..4a531db --- /dev/null +++ b/libcrange/source/src/perl_functions.c @@ -0,0 +1,227 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include "perl_functions.h" +#include + +#include +#include +#include + +#define _QUOTEME(x) #x +#define QUOTEME(x) _QUOTEME(x) + +#ifndef MODULE_PATH + #define MODULE_PATH /var/libcrange/perl +#endif + +#define PERLBOOT \ + "use strict;" \ + "use lib qw(" QUOTEME(MODULE_PATH) ");" \ + "our %_FUNCTIONS;" \ + "sub ::libcrange_load_file { my ( $lib, $prefix, $module ) = @_;" \ + " require qq($module.pm);" \ + " my @functions = $module->functions_provided;" \ + " my @mapped_functions = map { qq($prefix$_) } @functions; " \ + " for (@functions) { $_FUNCTIONS{qq($prefix$_)} = \\&{qq(${module}::$_)}; }" \ + " return @mapped_functions;" \ + "}" \ + "sub ::libcrange_call_func {" \ + " my $rr = shift;" \ + " my $func = shift;" \ + " my @args = @_;" \ + " my $fun_ref = $_FUNCTIONS{$func};" \ + " die qq(No function $func\\n) unless $fun_ref;" \ + " return $fun_ref->($rr, @args);" \ + "}" + +static void lr_init_shared_libs(pTHX); +static PerlInterpreter* perl_interp = NULL; + +static const char** +get_exported_functions(libcrange* lr, apr_pool_t* pool, + const char* module, const char* prefix) +{ + int i, count; + const char** functions; + dSP; + + ENTER; + SAVETMPS; + PUSHMARK(SP); + XPUSHs(sv_2mortal(newSViv((IVTYPE)lr))); + XPUSHs(sv_2mortal(newSVpv(prefix, 0))); + XPUSHs(sv_2mortal(newSVpv(module, 0))); + PUTBACK; + + count = call_pv("::libcrange_load_file", G_EVAL | G_ARRAY); + + SPAGAIN; + + if (SvTRUE(ERRSV)) { + fprintf(stderr, "Calling ::libcrange_load_file: %s", SvPV_nolen(ERRSV)); + functions = NULL; + } + else { + functions = apr_palloc(pool, sizeof(char*) * (count + 1)); + functions[count] = NULL; + + for (i=0; inodes->members; + av_unshift(array, n); + + nodes = range_get_hostnames(pool, r); + for (i=0; i +#include +#include +#include +#include +#include "range.h" +#include "range_request.h" +#include "set.h" +#include "range_parser.h" +#include "range_scanner.h" +#include "perl_functions.h" +#include "ast.h" +#include "range_request.h" + +int yyparse(void*); + +static range_request* current_rr; +void yyerror(const char* s) +{ + range_request_warn(current_rr, "%s", s); +} + +void range_destroy(range* r) +{ + set_destroy(r->nodes); +} + +range* range_from_set(range_request* rr, set* s) +{ + apr_pool_t* p = range_request_pool(rr); + range* r = apr_palloc(p, sizeof(range)); + r->nodes = s; + r->quoted = 0; + return r; +} + +range* copy_range(apr_pool_t* pool, const range* r) +{ + range* new_range = apr_palloc(pool, sizeof(range)); + new_range->quoted = r->quoted; + new_range->nodes = set_new(pool, 0); + set_union_inplace(new_range->nodes, r->nodes); + return new_range; +} + +range* do_range_expand(range_request* rr, const char* text) +{ + yyscan_t scanner; + struct range_extras extra; + int result; + + if (text == NULL) { + range* r = range_new(rr); + range_request_set(rr, r); + return r; + } + current_rr = rr; + extra.rr = rr; + + yylex_init(&scanner); + yyset_extra(&extra, scanner); + yy_scan_string(text, scanner); + result = yyparse(scanner); + yylex_destroy(scanner); + current_rr = NULL; + + if (result != 0) { + range* r = range_new(rr); + range_request_warn(rr, "parsing [%s]", text); + range_request_set(rr, r); + return r; + } + + range_request_set(rr, range_evaluate(rr, extra.theast)); + return range_request_results(rr); +} + +range* range_add(range* r, const char* text) +{ + assert(text); + set_add(r->nodes, text, NULL); + return r; +} + +range* range_remove(range* r, const char* text) +{ + set_remove(r->nodes, text); + return r; +} + +const char** range_get_hostnames(apr_pool_t* pool, const range* r) +{ + const char** ret; + set_element **members; + int i; + + ret = apr_palloc(pool, sizeof(char*) * (r->nodes->members + 1)); + members = set_members(r->nodes); + if (r->quoted) { + for (i = 0; members[i]; i++) + ret[i] = apr_psprintf(pool, "\"%s\"", members[i]->name); + ret[i] = NULL; + } else { + for (i = 0; members[i]; i++) + ret[i] = members[i]->name; + ret[i] = NULL; + } + return ret; +} + +range* range_from_hostnames(range_request* rr, + const char** names) +{ + range* r; + int i; + + r = range_new(rr); + for(i = 0; names[i]; i++) + range_add(r, names[i]); + + return r; +} + +range* range_new(range_request* rr) +{ + apr_pool_t* pool = range_request_pool(rr); + range* r = apr_palloc(pool, sizeof(range)); + r->nodes = set_new(pool, 0); + r->quoted = 0; + return r; +} + +range* range_from_null(range_request* rr) +{ + return range_new(rr); +} + +range* range_from_match(range_request* rr, + const range* r, const char* regex) +{ + range* ret; + int i; + int err_offset; + int ovector[30]; + int count; + const char* error; + const char** members; + pcre* re; + apr_pool_t* pool = range_request_pool(rr); + + members = range_get_hostnames(pool, r); + ret = range_new(rr); + + re = pcre_compile(regex, 0, &error, &err_offset, NULL); + if (!re) { + range_request_warn(rr, "regex [%s] [%s]", regex, error); + return ret; + } + + for (i = 0; members[i]; i++) { + count = pcre_exec(re, NULL, members[i], + strlen(members[i]), 0, 0, ovector, 30); + if (count > 0) /* it matched */ + range_add(ret, members[i]); + } + pcre_free(re); + + return ret; +} + +range* range_from_literal(range_request* rr, + const char* literal) +{ + range* r = range_new(rr); + range_add(r, literal); + return r; +} + +range* range_from_nonrange_literal(range_request* rr, + const char* literal) +{ + range* r = range_new(rr); + range_add(r, literal); + r->quoted = 1; + return r; +} + +range* range_from_braces(range_request* rr, + const range* r1, const range* r2, const range* r3) +{ + int i, j, k; + set_element** m1; + set_element** m2; + set_element** m3; + set* temp = NULL; + range* bigrange; + char* bundle; + apr_pool_t* pool = range_request_pool(rr); + + if(r1->nodes->members == 0) { + if(!temp) { + temp = set_new(pool, 1); + set_add(temp, "", NULL); + } + m1 = set_members(temp); + } else m1 = set_members(r1->nodes); + + if(r2->nodes->members == 0) { + if(!temp) { + temp = set_new(pool, 1); + set_add(temp, "", NULL); + } + m2 = set_members(temp); + } else m2 = set_members(r2->nodes); + + if(r3->nodes->members == 0) { + if(!temp) { + temp = set_new(pool, 1); + set_add(temp, "", NULL); + } + m3 = set_members(temp); + } else m3 = set_members(r3->nodes); + + bigrange = range_new(rr); + + for(i = 0; m1[i]; i++) + for(j = 0; m2[j]; j++) + for(k = 0; m3[k]; k++) { + bundle = apr_pstrcat(pool, + m1[i]->name, m2[j]->name, + m3[k]->name, NULL); + range_add(bigrange, bundle); + } + + if (temp) set_destroy(temp); + bigrange->quoted = r1->quoted || r2->quoted || r3->quoted; + return bigrange; +} + +range* range_from_union(range_request* rr, + const range* r1, const range* r2) +{ + range* r3 = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + + r3->nodes = set_union(pool, r1->nodes, r2->nodes); + r3->quoted = r1->quoted || r2->quoted; + return r3; +} + +void range_union_inplace(range_request* rr, + range* dst, const range* src) +{ + set_union_inplace(dst->nodes, src->nodes); +} + +range* range_from_inter(range_request* rr, + const range* r1, const range* r2) +{ + range* r3 = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + + r3->nodes = set_intersect(pool, r1->nodes, r2->nodes); + r3->quoted = r1->quoted || r2->quoted; + return r3; +} + +void range_diff_inplace(range_request* rr, + range* dst, const range* r2) +{ + set_diff_inplace(dst->nodes, r2->nodes); +} + +range* range_from_diff(range_request* rr, + const range* r1, const range* r2) +{ + range* r3 = range_new(rr); + apr_pool_t* pool = range_request_pool(rr); + + r3->nodes = set_diff(pool, r1->nodes, r2->nodes); + r3->quoted = r1->quoted || r2->quoted; + return r3; +} + +static rangeparts *rangeparts_new(apr_pool_t* pool) +{ + rangeparts* rp; + + rp = apr_palloc(pool, sizeof(rangeparts)); + rp->prefix = NULL; + rp->first = 0; + rp->last = 0; + rp->domain = NULL; + + return rp; +} + +rangeparts *rangeparts_from_hostname(range_request* rr, + const char* hostname) +{ + pcre* re; + const char* error; + rangeparts* rangeparts; + int offset, count; + int offsets[128]; + static pcre* regex_node = NULL; + apr_pool_t* pool = range_request_pool(rr); + + if (!regex_node) + regex_node = pcre_compile(NODE_RE, 0, &error, &offset, NULL); + + re = regex_node; + count = pcre_exec(re, NULL, hostname, strlen(hostname), + 0, 0, offsets, sizeof(offsets)/sizeof(int)); + if (count > 0) { + /* + * 1 == prefix + * 2 == range start + * 3 == domain, maybe + * 4 == range specifier: - or .. + * 5 == range end + * 6 == domain, maybe + */ + rangeparts = rangeparts_new(pool); + + rangeparts->prefix = libcrange_get_pcre_substring(pool, hostname, offsets, 1); + rangeparts->first = libcrange_get_pcre_substring(pool, hostname, offsets, 2); + rangeparts->last = libcrange_get_pcre_substring(pool, hostname, offsets, 5); + + if ((offsets[7] - offsets[6]) > 0) { + /* if we have a domain */ + rangeparts->domain = libcrange_get_pcre_substring(pool, hostname, offsets, 3); + } else if ((offsets[13] - offsets[12]) > 0) { + rangeparts->domain = libcrange_get_pcre_substring(pool, hostname, offsets, 6); + } else { + rangeparts->domain = ""; + } + /* + if (hostname[offsets[8]] == '-') { + range_request_warn_type(rr, "DEPRECATED_SYNTAX", hostname); + } + */ + + return rangeparts; + } + + return NULL; +} + +range* range_from_rangeparts(range_request* rr, + const rangeparts* parts) +{ + int i; + int f, l, firstlength, lastlength, length; + range* r; + char* pad1 = ""; + char* first; + char* last; + char* tmpstr; + apr_pool_t* pool = range_request_pool(rr); + + r = range_new(rr); + + firstlength = strlen(parts->first); + lastlength = strlen(parts->last); + first = parts->first; + last = parts->last; + + if (firstlength > lastlength) { + pad1 = apr_palloc(pool, firstlength - lastlength + 1); + for(i=0; i < (firstlength - lastlength); i++) + pad1[i] = parts->first[i]; + pad1[i] = '\0'; + first = parts->first + i; + } + + f = atoi(first); + l = atoi(last); + + length = firstlength > lastlength ? lastlength : firstlength; + + for(i=f; i<=l; i++) { + tmpstr = apr_psprintf(pool, "%s%s%0*d%s", + parts->prefix, pad1, length, i, parts->domain); + range_add(r, tmpstr); + } + + return r; +} + +range* range_from_group(range_request* rr, + const range* r) +{ + const range* rl[2]; + range* ret; + + rl[0] = r; + rl[1] = NULL; + + ret = range_from_function(rr, "group", rl); + + return ret; +} + +range* range_from_function(range_request* rr, + const char* funcname, const range** r) +{ + range* ret; + range* (*f)(range_request*, const range**); + const char* perl_module; + libcrange* lr = range_request_lr(rr); + + perl_module = libcrange_get_perl_module(lr, funcname); + if (perl_module) + ret = perl_function(rr, funcname, r); + else { + f = libcrange_get_function(lr, funcname); + if (!f) { + range_request_warn_type(rr, "NO_FUNCTION", funcname); + return range_new(rr); + } + ret = (*f)(rr, r); + } + return ret; +} diff --git a/libcrange/source/src/range.h b/libcrange/source/src/range.h new file mode 100644 index 0000000..96dbf7b --- /dev/null +++ b/libcrange/source/src/range.h @@ -0,0 +1,100 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef RANGE_H +#define RANGE_H + +#include "libcrange.h" +#include "range_request.h" +#include "set.h" + +#define NODE_RE "^" \ + /* valid hostname chars for a prefix */ \ + "([-\\w.]*?)" \ + /* beginning of range */ \ + "(\\d+)" \ + /* an optional domain */ \ + "(\\.[-A-Za-z\\d.]*[-A-Za-z]+[-A-Za-z\\d.]*)?" \ + /* our range indicator: warn when using the deprecated - */ \ + "(-|\\.\\.)" \ + /* if they used a prefix, they can repeat it here */ \ + "\\1?" \ + /* the end of the range */ \ + "(\\d+)" \ + /* they can repeat the domain used before, or specify one now */ \ + "((?(3)\\3|(?:\\.[-A-Za-z\\d.]+)?))$" + +typedef struct rangeparts +{ + char* prefix; + char* first; + char* last; + char* domain; +} rangeparts; + +typedef struct range +{ + set* nodes; + int quoted; +} range; + +typedef struct range_extras +{ + struct range_request* rr; + char string_buf[32768]; + char* string_buf_ptr; + struct rangeast* theast; +} range_extras; + +typedef struct rangelist +{ + struct range* range; + struct rangelist* next; +} funcargs; + +#define range_members(r) ((r)->nodes->members) + +range* copy_range(apr_pool_t* pool, const range* r); +range* do_range_expand(range_request* rr, const char* text); +const char** range_get_hostnames(apr_pool_t* pool, const range* r); +range* range_new(range_request* rr); + +void range_union_inplace(range_request* rr, range* r1, const range* r2); +void range_diff_inplace(range_request* rr, range* r1, const range* r2); + +rangeparts* rangeparts_from_hostname(range_request* rr, const char* hostname); +range* range_add(range* r, const char* text); +range* range_remove(range* r, const char* text); + +range* range_from_match(range_request* rr, + const range* r, const char* regex); +range* range_from_hostnames(range_request* rr, + const char** names); +range* range_from_literal(range_request* rr, + const char* literal); +range* range_from_nonrange_literal(range_request* rr, + const char* literal); +range* range_from_rangeparts(range_request* rr, + const rangeparts* parts); +range* range_from_parens(range_request* rr, + const range* r); +range* range_from_union(range_request* rr, + const range* r1, const range* r2); +range* range_from_braces(range_request* rr, + const range* left, const range* center, + const range* right); +range* range_from_diff(range_request* rr, + const range* r1, const range* r2); +range* range_from_inter(range_request* rr, + const range* r1, const range* r2); +range* range_from_group(range_request* rr, + const range* r); +range* range_from_function(range_request* rr, + const char* funcname, const range** r); +range* range_from_set(range_request* rr, set* s); + +void range_destroy(range* r); + +#endif diff --git a/libcrange/source/src/range_compress.c b/libcrange/source/src/range_compress.c new file mode 100644 index 0000000..4bbd20d --- /dev/null +++ b/libcrange/source/src/range_compress.c @@ -0,0 +1,110 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include "range_sort.h" + +#include "range_compress.h" +#include "range.h" +#include "range_parts.h" +#include "range_request.h" + +#include +#include +#include + +static node_parts_int* init_prev_parts(apr_pool_t* pool) +{ + node_parts_int* result = apr_palloc(pool, sizeof(node_parts_int)); + result->prefix = result->domain = result->num_str = result->full_name = ""; + result->num = -2; + return result; +} + +static const char* ignore_common_prefix(apr_pool_t* pool, int n1, int n2) +{ + char* s1 = apr_itoa(pool, n1); + char* s2 = apr_itoa(pool, n2); + int n, len1, len2; + len1 = strlen(s1); + len2 = strlen(s2); + if (len1 < len2) return s2; + n = 0; + while (n < len1 && s1[n] == s2[n]) ++n; + return s2 + n; +} + +static const char* fmt_group(apr_pool_t* pool, node_parts_int* parts, int count) +{ + return apr_psprintf(pool, "%s%s-%s%s", parts->prefix, parts->num_str, + ignore_common_prefix(pool, parts->num, parts->num + count), + parts->domain); +} + +const char* do_range_compress(range_request* rr, const range* r) +{ + int i; + #define MAX_NUM_GROUPS 65536 + const char* groups[MAX_NUM_GROUPS]; + int num_groups = 0; + int count; + char* result; + char* presult; + int result_size; + const char** sorted_nodes; + int n = r->nodes->members; + apr_pool_t* pool = range_request_pool(rr); + node_parts_int* prev = init_prev_parts(pool); + int prev_num_str_len = -1; + + if (n == 0) return ""; + + sorted_nodes = do_range_sort(rr, r); + init_range_parts(); + + count = 0; + for (i=0; iprefix, prev->prefix) == 0 && + strcmp(parts->domain, prev->domain) == 0 && + (parts->num == prev->num + count + 1) && + strlen(parts->num_str) == prev_num_str_len) count++; + else { + if (*prev->full_name) { + if (count > 0) + groups[num_groups] = fmt_group(pool, prev, count); + else + groups[num_groups] = prev->full_name; + ++num_groups; + if (num_groups == MAX_NUM_GROUPS) { + range_request_warn(rr, "%s\n", "too many compressed groups"); + return ""; + } + } + prev = parts; + if (prev->num_str) + prev_num_str_len = strlen(prev->num_str); + count = 0; + } + } + if (count > 0) + groups[num_groups] = fmt_group(pool, prev, count); + else + groups[num_groups] = prev->full_name; + + /* num_groups is 1 less than the # of groups */ + result_size = num_groups; /* commas */ + for (i=0; i<=num_groups; ++i) result_size += strlen(groups[i]); + presult = result = apr_palloc(pool, result_size + 1); + + for (i=0; i +#include "range.h" +#include "ast.h" + +#define YYPARSE_PARAM scanner +#define YYLEX_PARAM scanner +#include "range_parser_defs.h" + +%} + +%pure-parser +%locations +%error-verbose +%start main + +%token tGROUP tUNION tDIFF tINTER tNOT tADMIN tGET_CLUSTER +%token tGET_GROUP tCLUSTER tCOLON tLPAREN tRPAREN tSEMI +%token tLBRACE tRBRACE tEOL tCOLON tERROR tHASH tEOF tWHITESPACE + +%token tLITERAL tREGEX tNONRANGE_LITERAL +%token tRANGEPARTS + +%left tREGEX +%left tUNION tDIFF tINTER +%left tNOT +%nonassoc tCLUSTER tGROUP tADMIN tGET_CLUSTER tGET_GROUP +%left tCOLON +%left tSEMI +%left tLBRACE tRBRACE + +%type rangeexpr +%type main +%type funcargs + +%union { + struct rangeast *rangeast; + struct rangeparts *rangeparts; + char *strconst; +}; + +%% +main : rangeexpr +{ + (*((range_extras**)scanner))->theast = $1; +}; + +rangeexpr : { + range_extras* e = *(range_extras**)scanner; + $$ = range_ast_new(range_request_pool(e->rr), AST_NOTHING); +} +| tREGEX +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_REGEX); + r->data.string = $1; + $$ = r; +} +| tLITERAL +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_LITERAL); + r->data.string = $1; + $$ = r; +} +| tNONRANGE_LITERAL +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_NONRANGE_LITERAL); + r->data.string = $1; + $$ = r; +} +| tRANGEPARTS +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_PARTS); + r->data.parts = $1; + $$ = r; +} +| tLPAREN rangeexpr tRPAREN +{ + $$ = $2; +} +| rangeexpr tUNION rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_UNION); + $1->next = $3; + r->children = $1; + $$ = r; +} +| rangeexpr tLBRACE rangeexpr tRBRACE rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_BRACES); + $1->next = $3; + $3->next = $5; + r->children = $1; + $$ = r; +} +| rangeexpr tDIFF rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_DIFF); + $1->next = $3; + r->children = $1; + $$ = r; +} +| rangeexpr tINTER rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_INTER); + $1->next = $3; + r->children = $1; + $$ = r; +} +| tCLUSTER rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = "cluster"; + r->children = $2; + $$ = r; +} +| tADMIN rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = "get_admin"; + r->children = $2; + $$ = r; +} +| tGROUP rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = "group"; + r->children = $2; + $$ = r; +} +| tGET_CLUSTER rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = "get_cluster"; + r->children = $2; + $$ = r; +} +| tNOT rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_NOT); + r->children = $2; + $$ = r; +} +| tGET_GROUP rangeexpr +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = "get_groups"; + r->children = $2; + $$ = r; +} +| tLITERAL tLPAREN funcargs tRPAREN +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = $1; + r->children = $3; + $$ = r; +} +| tHASH tLITERAL tLPAREN funcargs tRPAREN +{ + range_extras* e = *(range_extras**)scanner; + rangeast *r = range_ast_new(range_request_pool(e->rr), AST_FUNCTION); + r->data.string = $2; + r->children = $4; + $$ = r; +}; + +funcargs: rangeexpr +{ + $$ = $1; +} +| rangeexpr tSEMI funcargs +{ + $1->next = $3; + $$ = $1; +}; + + + + +%% diff --git a/libcrange/source/src/range_parser_defs.h b/libcrange/source/src/range_parser_defs.h new file mode 100644 index 0000000..567bcfc --- /dev/null +++ b/libcrange/source/src/range_parser_defs.h @@ -0,0 +1,29 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef RANGE_PARSER_DEFS_H +#define RANGE_PARSER_DEFS_H + +typedef union YYSTYPE { + struct rangeast *rangeast; + struct rangeparts *rangeparts; + char *strconst; +} YYSTYPE; +#define YYSTYPE_IS_DECLARED 1 + +typedef struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +} YYLTYPE; +#define YYLTYPE_IS_DECLARED 1 + +void yyerror(const char* s); +int yylex(YYSTYPE* yylval_param, YYLTYPE* yylloc_param, void* yyscanner); + + +#endif diff --git a/libcrange/source/src/range_parts.c b/libcrange/source/src/range_parts.c new file mode 100644 index 0000000..d77cbbe --- /dev/null +++ b/libcrange/source/src/range_parts.c @@ -0,0 +1,46 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include +#include "range_parts.h" +#include "libcrange.h" +#include + +static pcre* num_node_re = 0; + +node_parts_int* node_to_parts(apr_pool_t* pool, const char* node_name) +{ + int offsets[64]; + int count; + node_parts_int* result = apr_palloc(pool, sizeof(node_parts_int)); + result->full_name = node_name; + count = pcre_exec(num_node_re, NULL, node_name, strlen(node_name), + 0, 0, offsets, sizeof(offsets) / sizeof(int)); + if (count > 0) { + result->prefix = libcrange_get_pcre_substring(pool, node_name, offsets, 1); + result->num_str = libcrange_get_pcre_substring(pool, node_name, offsets, 2); + result->num = atoi(result->num_str); + result->domain = count > 3 ? libcrange_get_pcre_substring(pool, node_name, offsets, 3) : ""; + } + else { + result->prefix = ""; + result->domain = ""; + result->num_str = NULL; + result->num = 0; + } + return result; +} + +void init_range_parts(void) +{ + if (!num_node_re) { + int offsets[20]; + const char* error; + num_node_re = pcre_compile(NUMBERED_NODE_RE, 0, &error, offsets, NULL); + } + assert(num_node_re); +} diff --git a/libcrange/source/src/range_parts.h b/libcrange/source/src/range_parts.h new file mode 100644 index 0000000..0bae2cc --- /dev/null +++ b/libcrange/source/src/range_parts.h @@ -0,0 +1,25 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef RANGE_PARTS_H +#define RANGE_PARTS_H + +#include + +typedef struct node_parts_int +{ + const char* prefix; + const char* domain; + int num; + const char* num_str; + const char* full_name; +} node_parts_int; + +node_parts_int* node_to_parts(apr_pool_t* pool, const char* node_name); +void init_range_parts(void); + +#define NUMBERED_NODE_RE "^([-\\w.]*?)(\\d+)(\\.[-A-Za-z\\d.]*[-A-Za-z]+[-A-Za-z\\d.]*)?$" + +#endif diff --git a/libcrange/source/src/range_request.c b/libcrange/source/src/range_request.c new file mode 100644 index 0000000..a46d2c5 --- /dev/null +++ b/libcrange/source/src/range_request.c @@ -0,0 +1,164 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include + +#include "range_request.h" +#include "range.h" +#include "libcrange.h" +#include "set.h" +#include "range_compress.h" + +struct range_request { + apr_pool_t* pool; + char* warnings; + set* warn_type; + int warn_enabled; + struct libcrange* lr; + range* r; +}; + +range_request* range_request_new(struct libcrange* lr, apr_pool_t* pool) +{ + range_request* res = apr_palloc(pool, sizeof(range_request)); + res->lr = lr; + res->pool = pool; + res->warnings = NULL; + res->warn_type = NULL; + res->warn_enabled = 1; + res->r = NULL; + + return res; +} + +int range_request_warn_enabled(range_request* rr) +{ + return rr->warn_enabled; +} + +void range_request_disable_warns(range_request* rr) +{ + rr->warn_enabled = 0; +} + +void range_request_enable_warns(range_request* rr) +{ + rr->warn_enabled = 1; +} + +const char* range_request_compressed(range_request* rr) +{ + return do_range_compress(rr, rr->r); +} + +range* range_request_results(range_request* rr) +{ + return rr->r; +} + + +libcrange* range_request_lr(range_request* rr) +{ + return rr->lr; +} + +int range_request_has_warnings(range_request* rr) +{ + return rr->warnings || rr->warn_type; +} + +const char* range_request_warnings(range_request* rr) +{ + /* for each of the different warning types, create a + * warning of the type NO_CLUSTER: ks321-9 | NO_IP: pain,haides + */ + char* result = NULL; + if (!rr->warnings && !rr->warn_type) return ""; + + if (rr->warn_type) { + set_element** members = set_members(rr->warn_type); + while (*members) { + if (result) + result = apr_psprintf(rr->pool, "%s | %s: %s", result, + (*members)->name, + do_range_compress(rr, (*members)->data)); + else + result = apr_psprintf(rr->pool, "%s: %s", (*members)->name, + do_range_compress(rr, (*members)->data)); + members++; + } + } + + if (!rr->warnings) return result; + + if (result) + result = apr_psprintf(rr->pool, "%s | %s", result, rr->warnings); + else + result = rr->warnings; + + return result; +} + +const char** range_request_nodes(range_request* rr) +{ + assert(rr->r); + return range_get_hostnames(rr->pool, rr->r); +} + +void range_request_warn(range_request* rr, const char* fmt, ...) +{ + va_list ap; + char* p = rr->warnings; + char* warn; + + if (!rr->warn_enabled) return; + + va_start(ap, fmt); + warn = apr_pvsprintf(rr->pool, fmt, ap); + va_end(ap); + + if (p) + rr->warnings = apr_psprintf(rr->pool, "%s|%s", p, warn); + else + rr->warnings = warn; +} + +void range_request_warn_type(range_request* rr, const char* type, const char* node) +{ + range* nodes; + + if (!rr->warn_enabled) return; + + if (!rr->warn_type) + rr->warn_type = set_new(rr->pool, 0); + + /* nodes that generated a particular warning type */ + nodes = set_get_data(rr->warn_type, type); + + if (!nodes) { + nodes = range_new(rr); + set_add(rr->warn_type, type, nodes); + } + + range_add(nodes, node); +} + +apr_pool_t* range_request_pool(range_request* rr) +{ + return rr->pool; +} + +apr_pool_t* range_request_lr_pool(range_request* rr) +{ + return libcrange_get_pool(rr->lr); +} + +void range_request_set(range_request* rr, range* r) +{ + rr->r = r; +} + diff --git a/libcrange/source/src/range_request.h b/libcrange/source/src/range_request.h new file mode 100644 index 0000000..1c5ca11 --- /dev/null +++ b/libcrange/source/src/range_request.h @@ -0,0 +1,29 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef _RANGE_REQUEST_H +#define _RANGE_REQUEST_H + +#include "libcrange.h" + +/* range request interface to be used by modules and + * internal libcrange functions */ + +struct range; + +range_request* range_request_new(libcrange* lr, apr_pool_t* pool); +void range_request_warn(range_request* rr, const char* fmt, ...); +void range_request_warn_type(range_request* rr, const char* type, const char* node); +int range_request_warn_enabled(range_request* rr); +void range_request_disable_warns(range_request* rr); +void range_request_enable_warns(range_request* rr); + +apr_pool_t* range_request_pool(range_request* rr); +apr_pool_t* range_request_lr_pool(range_request* rr); +void range_request_set(range_request* rr, struct range* r); +struct range* range_request_results(range_request* rr); + +#endif + diff --git a/libcrange/source/src/range_scanner.l b/libcrange/source/src/range_scanner.l new file mode 100644 index 0000000..a0369de --- /dev/null +++ b/libcrange/source/src/range_scanner.l @@ -0,0 +1,151 @@ +%{ +#include +#include +#include +#include "libcrange.h" +#include "range.h" +#include "range_parser.h" + +#define YY_EXTRA_TYPE range_extras * +%} + +%option reentrant bison-locations bison-bridge +%option batch noyywrap +%option stack warn nounput +%option header="range_scanner.h" +%x regex +%x quote +%x singlequote +%x doublequote + +%% + +[ \t] /* ignore */; +\n return tEOL; + +\/ { + yy_push_state(regex, yyscanner); + yyextra->string_buf_ptr = yyextra->string_buf; +} +\\(\/|\\) { + *(yyextra->string_buf_ptr++) = yytext[1]; +} +\\(.|\n) { + *(yyextra->string_buf_ptr++) = '\\'; + *(yyextra->string_buf_ptr++) = yytext[1]; +} +[^\\/]+ { + int n = strlen(yytext); + memcpy(yyextra->string_buf_ptr, yytext, n); + yyextra->string_buf_ptr += n; +} +\/ { + yy_top_state(yyscanner); + yy_pop_state(yyscanner); + *(yyextra->string_buf_ptr) = '\0'; + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), + yyextra->string_buf); + return tREGEX; +} +q\( { + yy_push_state(quote, yyscanner); + yyextra->string_buf_ptr = yyextra->string_buf; +} +\\(.|\n) { + *(yyextra->string_buf_ptr++) = yytext[1]; +} +[^\\)]+ { + int n = strlen(yytext); + memcpy(yyextra->string_buf_ptr, yytext, n); + yyextra->string_buf_ptr += n; +} +\) { + yy_top_state(yyscanner); + yy_pop_state(yyscanner); + *(yyextra->string_buf_ptr) = '\0'; + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), + yyextra->string_buf); + return tNONRANGE_LITERAL; +} + + +\' { + yy_push_state(singlequote, yyscanner); + yyextra->string_buf_ptr = yyextra->string_buf; +} +\\(.|\n) { + *(yyextra->string_buf_ptr++) = yytext[1]; +} +[^\\']+ { + int n = strlen(yytext); + memcpy(yyextra->string_buf_ptr, yytext, n); + yyextra->string_buf_ptr += n; +} +\' { + yy_top_state(yyscanner); + yy_pop_state(yyscanner); + *(yyextra->string_buf_ptr) = '\0'; + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), + yyextra->string_buf); + return tNONRANGE_LITERAL; +} + +\" { + yy_push_state(doublequote, yyscanner); + yyextra->string_buf_ptr = yyextra->string_buf; +} +\\(.|\n) { + *(yyextra->string_buf_ptr++) = yytext[1]; +} +[^\\"]+ { + int n = strlen(yytext); + memcpy(yyextra->string_buf_ptr, yytext, n); + yyextra->string_buf_ptr += n; +} +\" { + yy_top_state(yyscanner); + yy_pop_state(yyscanner); + *(yyextra->string_buf_ptr) = '\0'; + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), + yyextra->string_buf); + return tNONRANGE_LITERAL; +} + + + +[a-zA-Z0-9_\.:][a-zA-Z0-9_\.:\-]+ { + rangeparts* r; + if((r = rangeparts_from_hostname(yyextra->rr, yytext))) + { + yylval->rangeparts = r; + return tRANGEPARTS; + } else { + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), yytext); + return tLITERAL; + } +} + +[a-zA-Z0-9_\.:]+ { + yylval->strconst = apr_pstrdup(range_request_pool(yyextra->rr), yytext); + return tLITERAL; +} + +",-" return tDIFF; +",&" return tINTER; +"&" return tINTER; +"," return tUNION; +"(" return tLPAREN; +")" return tRPAREN; +"{" return tLBRACE; +"}" return tRBRACE; +"^" return tADMIN; +"#" return tHASH; +"%" return tCLUSTER; +"@" return tGROUP; +"*" return tGET_CLUSTER; +"!" return tNOT; +"?" return tGET_GROUP; +"-" return tDIFF; +";" return tSEMI; + +%% diff --git a/libcrange/source/src/range_sort.c b/libcrange/source/src/range_sort.c new file mode 100644 index 0000000..49881bc --- /dev/null +++ b/libcrange/source/src/range_sort.c @@ -0,0 +1,54 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include "range_sort.h" +#include "range_parts.h" +#include "range_request.h" +#include "range.h" + +#include +#include +#include + +static int compare_parts(const node_parts_int** ptr_a, const node_parts_int** ptr_b) +{ + int pre, domain, num; + const node_parts_int* a = *ptr_a; + const node_parts_int* b = *ptr_b; + + pre = strcmp(a->prefix, b->prefix); + if (pre) return pre; + + domain = strcmp(a->domain, b->domain); + if (domain) return domain; + + num = a->num - b->num; + if (num) return num; + + return strcmp(a->full_name, b->full_name); +} + +const char** do_range_sort(range_request* rr, const range* r) +{ + const char** result; + int n = r->nodes->members; + apr_pool_t* pool = range_request_pool(rr); + node_parts_int** parts = apr_palloc(pool, n * sizeof(node_parts_int*)); + set_element** members = set_members(r->nodes); + int i; + + init_range_parts(); + for (i=0; iname); + + qsort(parts, n, sizeof(node_parts_int*), + (int (*) (const void*, const void*)) compare_parts); + + result = apr_palloc(pool, sizeof(char*) * (n + 1)); + for (i=0; ifull_name; + result[n] = NULL; + + return result; +} diff --git a/libcrange/source/src/range_sort.h b/libcrange/source/src/range_sort.h new file mode 100644 index 0000000..06d49f0 --- /dev/null +++ b/libcrange/source/src/range_sort.h @@ -0,0 +1,14 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef RANGE_SORT_H +#define RANGE_SORT_H + +struct range; +struct range_request; + +const char** do_range_sort(struct range_request* rr, const struct range *r); + +#endif /* RANGE_SORT_H */ diff --git a/libcrange/source/src/set.c b/libcrange/source/src/set.c new file mode 100644 index 0000000..ea340f3 --- /dev/null +++ b/libcrange/source/src/set.c @@ -0,0 +1,445 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include +#include +#include + +#include "set.h" +#include + +#define NUM_PRIMES 29 +static const unsigned long prime_list[NUM_PRIMES] = +{ + 19ul, 53ul, 97ul, 193ul, 389ul, 769ul, + 1543ul, 3079ul, 6151ul, 12289ul, 24593ul, + 49157ul, 98317ul, 196613ul, 393241ul, 786433ul, + 1572869ul, 3145739ul, 6291469ul, 12582917ul, 25165843ul, + 50331653ul, 100663319ul, 201326611ul, 402653189ul, 805306457ul, + 1610612741ul, 3221225473ul, 4294967291ul +}; + + +unsigned long next_prime(unsigned long n) +{ + const unsigned long* prime = prime_list; + assert(n < prime_list[NUM_PRIMES - 1]); + while (*prime < n) ++prime; + return *prime; +} + +static size_t _count_members(const set* s) +{ + size_t count = 0; + int i; + set_element* n; + + for (i = 0; i < s->hashsize; i++) + for (n = s->table[i]; n; n = n->next) + count++; + return count; +} + +set* set_new(apr_pool_t* parent_pool, int hashsize) +{ + set* s; + int i; + apr_pool_t* pool; + apr_pool_create(&pool, parent_pool); + s = apr_palloc(pool, sizeof(set)); + + s->pool = pool; + s->hashsize = next_prime(hashsize); + s->table = (set_element**) apr_palloc(pool, sizeof(set_element *) * + s->hashsize); + + for (i = 0; i < s->hashsize; i++) + s->table[i] = NULL; + + s->members = 0; + return s; +} + +void set_destroy(set* s) +{ + apr_pool_destroy(s->pool); +} + +static set_element* +set_element_new(apr_pool_t* pool, const char* name, void* data) +{ + set_element* n; + n = apr_palloc(pool, sizeof(set_element)); + n->name = apr_pstrdup(pool, name); + n->data = data; + n->next = NULL; + return n; +} + +#define HSIEH_HASH 1 + +#if defined(HSIEH_HASH) +static uint32_t string_hash(const char* data) +{ +#undef get16bits +#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ + || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) +#define get16bits(d) (*((const uint16_t *) (d))) +#endif + +#if !defined (get16bits) +#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8)\ + +(uint32_t)(((const uint8_t *)(d))[0]) ) +#endif + uint32_t len = strlen(data); + uint32_t hash = len, tmp; + int rem; + + if (len <= 0 || data == NULL) return 0; + + rem = len & 3; + len >>= 2; + + /* Main loop */ + for (;len > 0; len--) { + hash += get16bits (data); + tmp = (get16bits (data+2) << 11) ^ hash; + hash = (hash << 16) ^ tmp; + data += 2*sizeof (uint16_t); + hash += hash >> 11; + } + + /* Handle end cases */ + switch (rem) { + case 3: hash += get16bits (data); + hash ^= hash << 16; + hash ^= data[sizeof (uint16_t)] << 18; + hash += hash >> 11; + break; + case 2: hash += get16bits (data); + hash ^= hash << 11; + hash += hash >> 17; + break; + case 1: hash += *data; + hash ^= hash << 10; + hash += hash >> 1; + } + + /* Force "avalanching" of final 127 bits */ + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + + return hash; +} + +#elif defined(PJW_HASH) +/* PJW hash function (optmized for 32bit ints) */ +static unsigned string_hash(const char* str) +{ + unsigned hash = 0; + assert(str); + while (*str) { + unsigned x; + hash = (hash << 4) + *str; + if ((x = hash & 0xf0000000) != 0) + hash ^= x >> 24; + hash &= ~x; + ++str; + } + return hash; +} +#elif defined(RS_HASH) +/* Robert Sedgewick hash function */ +static unsigned string_hash(const char* str) +{ + unsigned b = 378551; + unsigned a = 63689; + unsigned hash = 0; + unsigned i = 0; + + while (*str) { + hash = hash * a + (*str); + a = a * b; + ++str; + } + + return hash; +} +#elif defined(DJB_HASH) +/* DJB's extremely efficient hash function */ +static unsigned string_hash(const char* str) +{ + unsigned int hash = 5381; + while (*str) + hash = ((hash << 5) + hash) + (*str++); + + return hash; + +} +#elif defined(DEK_HASH) +static unsigned string_hash(const char* str) +{ + unsigned len = strlen(str); + unsigned hash = len; + unsigned i = 0; + + for (i = 0; i < len; str++, i++) + { + hash = ((hash << 5) ^ (hash >> 27)) ^ (*str); + } + return hash; +} +#endif + +#define set_hash(s, str) (string_hash(str) % s->hashsize) + +static void resize(set* s, size_t num_elements_hint) +{ + size_t old_n = s->hashsize; + if (old_n < num_elements_hint) { + size_t n = next_prime(num_elements_hint); + if (n > old_n) { + int i; + set_element* *new_table = apr_pcalloc(s->pool, sizeof(set_element* ) * n); + for (i=0; itable[i]; bucket; bucket = next_bucket) { + int new_idx = string_hash(bucket->name) % n; + next_bucket = bucket->next; + bucket->next = new_table[new_idx]; + new_table[new_idx] = bucket; + + } + } + s->table = new_table; + s->hashsize = n; + } + } +} + +static set_element* set_add_noresize(set* s, const char* name, void* data) +{ + int i; + set_element* n; + i = set_hash(s, name); + for (n = s->table[i]; n; n = n->next) + if (!strcmp(n->name, name)) { + n->data = data; + return n; + } + + n = set_element_new(s->pool, name, data); + n->next = s->table[i]; + s->table[i] = n; + s->members++; + + return n; +} + +set_element* set_add(set* s, const char* name, void* data) +{ + resize(s, s->members + 1); + return set_add_noresize(s, name, data); +} + +set_element* set_get(const set* s, const char* name) +{ + int i; + set_element* n; + i = set_hash(s, name); + + for (n = s->table[i]; n; n = n->next) + if (!strcmp(n->name, name)) + return n; + + return NULL; +} +void* set_get_data(const set* s, const char* name) +{ + set_element* e = set_get(s, name); + + if (e) return e->data; + return NULL; +} + +set_element** set_members(const set* s) +{ + int i, j; + set_element* n; + set_element** ret; + + ret = apr_palloc(s->pool, sizeof(set_element* ) * (s->members + 1)); + + j = 0; + for (i = 0; i < s->hashsize; i++) + for (n = s->table[i]; n; n = n->next) + ret[j++] = n; + + ret[j] = NULL; + return ret; +} + +set* set_union(apr_pool_t* pool, const set* s1, const set* s2) +{ + set* s; + int i; + set_element* n; + + s = set_new(pool, s1->members + s2->members); + + for (i = 0; i < s1->hashsize; i++) + for (n = s1->table[i]; n; n = n->next) + set_add_noresize(s, n->name, n->data); + + for (i = 0; i < s2->hashsize; i++) + for (n = s2->table[i]; n; n = n->next) + set_add_noresize(s, n->name, n->data); + +#if defined(DEBUG_HASH) + dump_hash_values(s); +#endif + return s; +} + +void set_union_inplace(set* s, const set* s2) +{ + int i; + set_element* n; + + resize(s, s->members + s2->members); + for (i = 0; i < s2->hashsize; i++) + for (n = s2->table[i]; n; n = n->next) + set_add_noresize(s, n->name, n->data); + + assert(s->members == _count_members(s)); +#if defined(DEBUG_HASH) + dump_hash_values(s); +#endif +} + +set* set_diff(apr_pool_t* pool, const set* s1, const set* s2) +{ + set* s; + int i; + set_element* n; + + s = set_new(pool, s1->hashsize); + + for (i = 0; i < s->hashsize; i++) + for (n = s1->table[i]; n; n = n->next) + if (!set_get(s2, n->name)) + set_add_noresize(s, n->name, n->data); + +#if defined(DEBUG_HASH) + dump_hash_values(s); +#endif + return s; +} + +void set_diff_inplace(set* s, const set* s2) +{ + int i; + set_element* n; + + for (i = 0; i < s->hashsize; i++) { + set_element* prev = NULL; + for (n = s->table[i]; n; n = n->next) { + if (set_get(s2, n->name)) { + if (prev) + prev->next = n->next; + else + s->table[i] = n->next; + + s->members--; + } + else prev = n; + } + } + +#if defined(DEBUG_HASH) + dump_hash_values(s); +#endif +} + +set* set_intersect(apr_pool_t* pool, const set* s1, const set* s2) +{ + int i; + set_element* n; + set* s; + + /* always loop over the set with fewer elements: s1 > s2 */ + if (s1->members < s2->members) { + /* swap */ + const set* tmp = s1; + s1 = s2; + s2 = tmp; + } + + s = set_new(pool, s1->hashsize); + + for (i = 0; i < s2->hashsize; i++) + for (n = s2->table[i]; n; n = n->next) + if (set_get(s1, n->name)) + set_add_noresize(s, n->name, n->data); + return s; +} + +set* set_remove(set* s, const char* name) +{ + set_element* n; + set_element* prev = NULL; + int i = set_hash(s, name); + + for (n = s->table[i]; n; prev = n, n = n->next) { + if (!strcmp(n->name, name)) { + if (prev) + prev->next = n->next; + else + s->table[i] = n->next; + s->members--; + return s; + } + } + + return s; +} + +char* set_dump(const set* s) +{ + set_element** memb; + int i; + + fprintf(stderr, "Members: %ld\n", (long)s->members); + memb = set_members(s); + for (i=0; memb[i]; i++) { + fprintf(stderr, " - %s\n", memb[i]->name); + } + return ""; +} + +#if defined(DEBUG_HASH) +void dump_hash_values(const set* s) +{ + int i; + int used = 0, max_chain = 0; + for (i=0; ihashsize; i++) { + int this_chain = 0; + set_element* n = s->table[i]; + if (n) used++; + while (n) { + this_chain++; + n = n->next; + } + max_chain = max_chain < this_chain ? this_chain : max_chain; + } + printf("DEBUG: %d %d %d %d\n", used, s->members, s->hashsize, max_chain); +} + +#endif + diff --git a/libcrange/source/src/set.h b/libcrange/source/src/set.h new file mode 100644 index 0000000..3f5a91b --- /dev/null +++ b/libcrange/source/src/set.h @@ -0,0 +1,42 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#ifndef SET_H +#define SET_H + +#include +#include + +typedef struct set_element +{ + const char* name; + void* data; + struct set_element* next; +} set_element; + +typedef struct set +{ + size_t hashsize; + struct set_element** table; + size_t members; + apr_pool_t* pool; +} set; + +char* set_dump(const set* s); +set_element* set_add(set* theset, const char* name, void* data); +set_element* set_get(const set* theset, const char* name); +void* set_get_data(const set* theset, const char* name); + +set_element** set_members(const set* s); +set* set_remove(set* theset, const char* name); +set* set_new(apr_pool_t* pool, int hashsize); +void set_destroy(set* s); +set* set_union(apr_pool_t* pool, const set* s1, const set* s2); +void set_union_inplace(set* s, const set* s2); +set* set_intersect(apr_pool_t* pool, const set* s1, const set* s2); +set* set_diff(apr_pool_t* pool, const set* s1, const set* s2); +void set_diff_inplace(set* s, const set* s2); + +#endif diff --git a/librange/index.yaml b/librange/index.yaml new file mode 100644 index 0000000..015b345 --- /dev/null +++ b/librange/index.yaml @@ -0,0 +1,11 @@ +--- +deb: + requires: + - libpcre4 +default: + name: librange + summary: C library for range operations + version: '1.1.0' +x86_64: + nobuild: 1 + preferarch: i386 diff --git a/librange/scripts/post.sh b/librange/scripts/post.sh new file mode 100644 index 0000000..2d1871b --- /dev/null +++ b/librange/scripts/post.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +ldconfig diff --git a/librange/source/Makefile b/librange/source/Makefile new file mode 100644 index 0000000..95bd722 --- /dev/null +++ b/librange/source/Makefile @@ -0,0 +1,41 @@ +OCAMLMAKEFILE = OCamlMakefile + +SOURCES = memoize.ml range_utils.ml parser.mly lexer.mll tinydns.ml \ + netmask.ml netblocks.ml admins.ml hosts_netblocks.ml evaluate.ml range.ml +RESULT = range +PACKS = pcre +LIBS = unix + +PATH = /usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin + +LDPATH=-L /usr/lib/ocaml/site-lib/pcre/ -L /usr/local/lib -L /usr/lib + +librange.so: camlcode.o librange.o + ld -shared --whole-archive -o librange.so camlcode.o librange.o /usr/lib/ocaml/libunix.a /usr/lib/ocaml/libasmrun.a /usr/lib/ocaml/site-lib/pcre/libpcre_stubs.a -lm -lpcre -ldl + + +# unused +#pcre_stubs.o: +# ar x /opt/godi/lib/ocaml/pkg-lib/pcre/libpcre_stubs.a +# unused +#libunix_unix.o: +# sh rip_objects_hack.sh /opt/godi/lib/ocaml/std-lib + +install: + mkdir -p $(DESTDIR)/usr/include/ $(DESTDIR)/usr/lib/ + install -m 644 range.h $(DESTDIR)/usr/include/ + install -m 755 librange.so $(DESTDIR)/usr/lib/ + +camlcode.o: ncl + ocamlopt -I /usr/lib/ocaml/site-lib/pcre -output-obj -o camlcode.o unix.cmxa pcre.cmxa range_utils.cmx parser.cmx lexer.cmx memoize.cmx tinydns.cmx netmask.cmx admins.cmx netblocks.cmx hosts_netblocks.cmx evaluate.cmx range.cmx + +librange.o: librange.c + gcc -Wall -g -c librange.c -I /usr/lib/ocaml + +cleaner: clean + rm -f librange.so librange.o testrange camlcode.o pcre_stubs.o + rm -rf a b + rm -f memoize.o memoize.cmx + +include $(OCAMLMAKEFILE) + diff --git a/librange/source/OCamlMakefile b/librange/source/OCamlMakefile new file mode 100644 index 0000000..de852ab --- /dev/null +++ b/librange/source/OCamlMakefile @@ -0,0 +1,1046 @@ +########################################################################### +# OCamlMakefile +# Copyright (C) 1999-2004 Markus Mottl +# +# For updates see: +# http://www.oefai.at/~markus/ocaml_sources +# +# $Id: OCamlMakefile,v 1.53 2004/05/04 19:08:39 mottl Exp $ +# +########################################################################### + +# Modified by damien for .glade.ml compilation + +# Set these variables to the names of the sources to be processed and +# the result variable. Order matters during linkage! + +ifndef SOURCES + SOURCES := foo.ml +endif +export SOURCES + +ifndef RES_CLIB_SUF + RES_CLIB_SUF := _stubs +endif +export RES_CLIB_SUF + +ifndef RESULT + RESULT := foo +endif +export RESULT + +ifndef DOC_FILES + DOC_FILES := $(filter %.mli, $(SOURCES)) +endif +export DOC_FILES + +export BCSUFFIX +export NCSUFFIX + +ifndef TOPSUFFIX + TOPSUFFIX := .top +endif +export TOPSUFFIX + +# Eventually set include- and library-paths, libraries to link, +# additional compilation-, link- and ocamlyacc-flags +# Path- and library information needs not be written with "-I" and such... +# Define THREADS if you need it, otherwise leave it unset (same for +# USE_CAMLP4)! + +export THREADS +export VMTHREADS +export ANNOTATE +export USE_CAMLP4 + +export INCDIRS +export LIBDIRS +export EXTLIBDIRS +export RESULTDEPS +export OCAML_DEFAULT_DIRS + +export LIBS +export CLIBS + +export OCAMLFLAGS +export OCAMLNCFLAGS +export OCAMLBCFLAGS + +export OCAMLLDFLAGS +export OCAMLNLDFLAGS +export OCAMLBLDFLAGS + +ifndef OCAMLCPFLAGS + OCAMLCPFLAGS := a +endif + +export OCAMLCPFLAGS + +export PPFLAGS + +export YFLAGS +export IDLFLAGS + +export OCAMLDOCFLAGS + +export DVIPSFLAGS + +export STATIC + +# Add a list of optional trash files that should be deleted by "make clean" +export TRASH + +#################### variables depending on your OCaml-installation + +ifdef MINGW + export MINGW + WIN32 := 1 +endif +ifdef MSVC + export MSVC + WIN32 := 1 + EXT_OBJ := obj + EXT_LIB := lib + ifeq ($(CC),gcc) + # work around GNU Make default value + ifdef THREADS + CC := cl -MT + else + CC := cl + endif + endif + ifeq ($(CXX),g++) + # work around GNU Make default value + CXX := $(CC) + endif + CFLAG_O := -Fo +endif +ifdef WIN32 + EXT_CXX := cpp + EXE := .exe +endif + +ifndef EXT_OBJ + EXT_OBJ := o +endif +ifndef EXT_LIB + EXT_LIB := a +endif +ifndef EXT_CXX + EXT_CXX := cc +endif +ifndef EXE + EXE := # empty +endif +ifndef CFLAG_O + CFLAG_O := -o # do not delete this comment (preserves trailing whitespace)! +endif + +export CC +export CXX +export CFLAGS +export CXXFLAGS +export LDFLAGS + +ifndef RPATH_FLAG + ifdef ELF_RPATH_FLAG + RPATH_FLAG := $(ELF_RPATH_FLAG) + else + RPATH_FLAG := -R + endif +endif +export RPATH_FLAG + +ifndef PIC_FLAGS + PIC_FLAGS := -fPIC -DPIC +endif +export PIC_FLAGS + +BCRESULT := $(addsuffix $(BCSUFFIX), $(RESULT)) +NCRESULT := $(addsuffix $(NCSUFFIX), $(RESULT)) +TOPRESULT := $(addsuffix $(TOPSUFFIX), $(RESULT)) + +ifndef OCAMLFIND + OCAMLFIND := ocamlfind +endif +export OCAMLFIND + +ifndef OCAMLC + OCAMLC := ocamlc +endif +export OCAMLC + +ifndef OCAMLOPT + OCAMLOPT := ocamlopt +endif +export OCAMLOPT + +ifndef OCAMLMKTOP + OCAMLMKTOP := ocamlmktop +endif +export OCAMLMKTOP + +ifndef OCAMLCP + OCAMLCP := ocamlcp +endif +export OCAMLCP + +ifndef OCAMLDEP + OCAMLDEP := ocamldep +endif +export OCAMLDEP + +ifndef OCAMLLEX + OCAMLLEX := ocamllex +endif +export OCAMLLEX + +ifndef OCAMLYACC + OCAMLYACC := ocamlyacc +endif +export OCAMLYACC + +ifndef OCAMLMKLIB + OCAMLMKLIB := ocamlmklib +endif +export OCAMLMKLIB + +ifndef OCAML_GLADECC + OCAML_GLADECC := lablgladecc2 +endif +export OCAML_GLADECC + +ifndef OCAML_GLADECC_FLAGS + OCAML_GLADECC_FLAGS := +endif +export OCAML_GLADECC_FLAGS + +ifndef CAMELEON_REPORT + CAMELEON_REPORT := report +endif +export CAMELEON_REPORT + +ifndef CAMELEON_REPORT_FLAGS + CAMELEON_REPORT_FLAGS := +endif +export CAMELEON_REPORT_FLAGS + +ifndef CAMELEON_ZOGGY + CAMELEON_ZOGGY := camlp4o pa_zog.cma pr_o.cmo +endif +export CAMELEON_ZOGGY + +ifndef CAMELEON_ZOGGY_FLAGS + CAMELEON_ZOGGY_FLAGS := +endif +export CAMELEON_ZOGGY_FLAGS + +ifndef CAMLIDL + CAMLIDL := camlidl +endif +export CAMLIDL + +ifndef CAMLIDLDLL + CAMLIDLDLL := camlidldll +endif +export CAMLIDLDLL + +ifndef NOIDLHEADER + MAYBE_IDL_HEADER := -header +endif +export NOIDLHEADER + +export NO_CUSTOM + +ifndef CAMLP4 + CAMLP4 := camlp4 +endif +export CAMLP4 + +ifdef PACKS + empty := + space := $(empty) $(empty) + comma := , + ifdef PREDS + PRE_OCAML_FIND_PREDICATES := $(subst $(space),$(comma),$(PREDS)) + PRE_OCAML_FIND_PACKAGES := $(subst $(space),$(comma),$(PACKS)) + OCAML_FIND_PREDICATES := -predicates $(PRE_OCAML_FIND_PREDICATES) + OCAML_DEP_PREDICATES := -syntax $(PRE_OCAML_FIND_PREDICATES) + OCAML_FIND_PACKAGES := $(OCAML_FIND_PREDICATES) -package $(PRE_OCAML_FIND_PACKAGES) + OCAML_DEP_PACKAGES := $(OCAML_DEP_PREDICATES) -package $(PRE_OCAML_FIND_PACKAGES) + else + OCAML_FIND_PACKAGES := -package $(subst $(space),$(comma),$(PACKS)) + OCAML_DEP_PACKAGES := + endif + OCAML_FIND_LINKPKG := -linkpkg + REAL_OCAMLFIND := $(OCAMLFIND) +endif + +export OCAML_FIND_PACKAGES +export OCAML_DEP_PACKAGES +export OCAML_FIND_LINKPKG +export REAL_OCAMLFIND + +ifndef OCAMLDOC + OCAMLDOC := ocamldoc +endif +export OCAMLDOC + +ifndef LATEX + LATEX := latex +endif +export LATEX + +ifndef DVIPS + DVIPS := dvips +endif +export DVIPS + +ifndef PS2PDF + PS2PDF := ps2pdf +endif +export PS2PDF + +ifndef OCAMLMAKEFILE + OCAMLMAKEFILE := OCamlMakefile +endif +export OCAMLMAKEFILE + +ifndef OCAMLLIBPATH + OCAMLLIBPATH := \ + $(shell $(OCAMLC) 2>/dev/null -where || echo /usr/local/lib/ocaml) +endif +export OCAMLLIBPATH + +ifndef OCAML_LIB_INSTALL + OCAML_LIB_INSTALL := $(OCAMLLIBPATH)/contrib +endif +export OCAML_LIB_INSTALL + +########################################################################### + +#################### change following sections only if +#################### you know what you are doing! + +# delete target files when a build command fails +.PHONY: .DELETE_ON_ERROR +.DELETE_ON_ERROR: + +# for pedants using "--warn-undefined-variables" +export MAYBE_IDL +export REAL_RESULT +export CAMLIDLFLAGS +export THREAD_FLAG +export RES_CLIB +export MAKEDLL +export ANNOT_FLAG +INCFLAGS := + +SHELL := /bin/sh + +MLDEPDIR := ._d +BCDIDIR := ._bcdi +NCDIDIR := ._ncdi + +FILTER_EXTNS := %.mli %.ml %.mll %.mly %.idl %.c %.$(EXT_CXX) %.rep %.zog %.glade + +FILTERED := $(filter $(FILTER_EXTNS), $(SOURCES)) +SOURCE_DIRS := $(filter-out ./, $(sort $(dir $(FILTERED)))) + +FILTERED_REP := $(filter %.rep, $(FILTERED)) +DEP_REP := $(FILTERED_REP:%.rep=$(MLDEPDIR)/%.d) +AUTO_REP := $(FILTERED_REP:.rep=.ml) + +FILTERED_ZOG := $(filter %.zog, $(FILTERED)) +DEP_ZOG := $(FILTERED_ZOG:%.zog=$(MLDEPDIR)/%.d) +AUTO_ZOG := $(FILTERED_ZOG:.zog=.ml) + +FILTERED_GLADE := $(filter %.glade, $(FILTERED)) +DEP_GLADE := $(FILTERED_GLADE:%.glade=$(MLDEPDIR)/%.d) +AUTO_GLADE := $(FILTERED_GLADE:.glade=.ml) + +FILTERED_ML := $(filter %.ml, $(FILTERED)) +DEP_ML := $(FILTERED_ML:%.ml=$(MLDEPDIR)/%.d) + +FILTERED_MLI := $(filter %.mli, $(FILTERED)) +DEP_MLI := $(FILTERED_MLI:.mli=.di) + +FILTERED_MLL := $(filter %.mll, $(FILTERED)) +DEP_MLL := $(FILTERED_MLL:%.mll=$(MLDEPDIR)/%.d) +AUTO_MLL := $(FILTERED_MLL:.mll=.ml) + +FILTERED_MLY := $(filter %.mly, $(FILTERED)) +DEP_MLY := $(FILTERED_MLY:%.mly=$(MLDEPDIR)/%.d) $(FILTERED_MLY:.mly=.di) +AUTO_MLY := $(FILTERED_MLY:.mly=.mli) $(FILTERED_MLY:.mly=.ml) + +FILTERED_IDL := $(filter %.idl, $(FILTERED)) +DEP_IDL := $(FILTERED_IDL:%.idl=$(MLDEPDIR)/%.d) $(FILTERED_IDL:.idl=.di) +C_IDL := $(FILTERED_IDL:%.idl=%_stubs.c) +ifndef NOIDLHEADER + C_IDL += $(FILTERED_IDL:.idl=.h) +endif +OBJ_C_IDL := $(FILTERED_IDL:%.idl=%_stubs.$(EXT_OBJ)) +AUTO_IDL := $(FILTERED_IDL:.idl=.mli) $(FILTERED_IDL:.idl=.ml) $(C_IDL) + +FILTERED_C_CXX := $(filter %.c %.$(EXT_CXX), $(FILTERED)) +OBJ_C_CXX := $(FILTERED_C_CXX:.c=.$(EXT_OBJ)) +OBJ_C_CXX := $(OBJ_C_CXX:.$(EXT_CXX)=.$(EXT_OBJ)) + +PRE_TARGETS += $(AUTO_MLL) $(AUTO_MLY) $(AUTO_IDL) $(AUTO_ZOG) $(AUTO_REP) $(AUTO_GLADE) + +ALL_DEPS := $(DEP_ML) $(DEP_MLI) $(DEP_MLL) $(DEP_MLY) $(DEP_IDL) $(DEP_ZOG) $(DEP_REP) $(DEP_GLADE) + +MLDEPS := $(filter %.d, $(ALL_DEPS)) +MLIDEPS := $(filter %.di, $(ALL_DEPS)) +BCDEPIS := $(MLIDEPS:%.di=$(BCDIDIR)/%.di) +NCDEPIS := $(MLIDEPS:%.di=$(NCDIDIR)/%.di) + +ALLML := $(filter %.mli %.ml %.mll %.mly %.idl %.rep %.zog %.glade, $(FILTERED)) + +IMPLO_INTF := $(ALLML:%.mli=%.mli.__) +IMPLO_INTF := $(foreach file, $(IMPLO_INTF), \ + $(basename $(file)).cmi $(basename $(file)).cmo) +IMPLO_INTF := $(filter-out %.mli.cmo, $(IMPLO_INTF)) +IMPLO_INTF := $(IMPLO_INTF:%.mli.cmi=%.cmi) + +IMPLX_INTF := $(IMPLO_INTF:.cmo=.cmx) + +INTF := $(filter %.cmi, $(IMPLO_INTF)) +IMPL_CMO := $(filter %.cmo, $(IMPLO_INTF)) +IMPL_CMX := $(IMPL_CMO:.cmo=.cmx) +IMPL_ASM := $(IMPL_CMO:.cmo=.asm) +IMPL_S := $(IMPL_CMO:.cmo=.s) + +OBJ_LINK := $(OBJ_C_IDL) $(OBJ_C_CXX) +OBJ_FILES := $(IMPL_CMO:.cmo=.$(EXT_OBJ)) $(OBJ_LINK) + +EXECS := $(addsuffix $(EXE), \ + $(sort $(TOPRESULT) $(BCRESULT) $(NCRESULT))) +ifdef WIN32 + EXECS += $(BCRESULT).dll $(NCRESULT).dll +endif + +CLIB_BASE := $(RESULT)$(RES_CLIB_SUF) +ifneq ($(strip $(OBJ_LINK)),) + RES_CLIB := lib$(CLIB_BASE).$(EXT_LIB) +endif + +ifndef MSVC + DLLSONAME := dll$(CLIB_BASE).so +endif + +NONEXECS := $(INTF) $(IMPL_CMO) $(IMPL_CMX) $(IMPL_ASM) $(IMPL_S) \ + $(OBJ_FILES) $(PRE_TARGETS) $(BCRESULT).cma $(NCRESULT).cmxa \ + $(NCRESULT).$(EXT_LIB) $(BCRESULT).cmi $(BCRESULT).cmo \ + $(NCRESULT).cmi $(NCRESULT).cmx $(NCRESULT).o \ + $(RES_CLIB) $(IMPL_CMO:.cmo=.annot) + +ifndef MSVC + ifndef STATIC + NONEXECS += $(DLLSONAME) + endif +endif + +ifndef LIBINSTALL_FILES + LIBINSTALL_FILES := $(RESULT).mli $(RESULT).cmi $(RESULT).cma \ + $(RESULT).cmxa $(RESULT).$(EXT_LIB) $(RES_CLIB) + ifndef MSVC + ifndef STATIC + ifneq ($(strip $(OBJ_LINK)),) + LIBINSTALL_FILES += $(DLLSONAME) + endif + endif + endif +endif + +export LIBINSTALL_FILES + +ifdef WIN32 + # some extra stuff is created while linking DLLs + NONEXECS += $(BCRESULT).$(EXT_LIB) $(BCRESULT).exp $(NCRESULT).exp +endif + +TARGETS := $(EXECS) $(NONEXECS) + +# If there are IDL-files +ifneq ($(strip $(FILTERED_IDL)),) + MAYBE_IDL := -cclib -lcamlidl +endif + +ifdef USE_CAMLP4 + CAMLP4PATH := \ + $(shell $(CAMLP4) -where 2>/dev/null || echo /usr/local/lib/camlp4) + INCFLAGS := -I $(CAMLP4PATH) + CINCFLAGS := -I$(CAMLP4PATH) +endif + +DINCFLAGS := $(INCFLAGS) $(SOURCE_DIRS:%=-I %) $(OCAML_DEFAULT_DIRS:%=-I %) +INCFLAGS := $(DINCFLAGS) $(INCDIRS:%=-I %) +CINCFLAGS += $(SOURCE_DIRS:%=-I%) $(INCDIRS:%=-I%) $(OCAML_DEFAULT_DIRS:%=-I%) +CLIBFLAGS += $(SOURCE_DIRS:%=-L%) $(LIBDIRS:%=-L%) \ + $(EXTLIBDIRS:%=-L%) \ + $(OCAML_DEFAULT_DIRS:%=-L%) +ifeq ($(ELF_RPATH),yes) +CLIBFLAGS += $(EXTLIBDIRS:%=-Wl,$(RPATH_FLAG)%) +endif + + +ifndef PROFILING + INTF_OCAMLC := $(OCAMLC) +else + ifndef THREADS + INTF_OCAMLC := $(OCAMLCP) -p $(OCAMLCPFLAGS) + else + # OCaml does not support profiling byte code + # with threads (yet), therefore we force an error. + ifndef REAL_OCAMLC + $(error Profiling of multithreaded byte code not yet supported by OCaml) + endif + INTF_OCAMLC := $(OCAMLC) + endif +endif + +COMMON_LDFLAGS := $(LDFLAGS:%=-ccopt %) $(SOURCE_DIRS:%=-ccopt -L%) \ + $(LIBDIRS:%=-ccopt -L%) $(EXTLIBDIRS:%=-ccopt -L%) \ + $(OCAML_DEFAULT_DIRS:%=-ccopt -L%) +ifeq ($(ELF_RPATH),yes) +COMMON_LDFLAGS += $(EXTLIBDIRS:%=-ccopt -Wl,$(RPATH_FLAG)%) +endif + +ifndef MSVC + CLIBS_OPTS := $(CLIBS:%=-cclib -l%) +else + # MSVC libraries do not have 'lib' prefix + CLIBS_OPTS := $(CLIBS:%=-cclib %.lib) +endif +ifneq ($(strip $(OBJ_LINK)),) + ifdef CREATE_LIB + OBJS_LIBS := -cclib -l$(CLIB_BASE) $(CLIBS_OPTS) $(MAYBE_IDL) + else + OBJS_LIBS := $(OBJ_LINK) $(CLIBS_OPTS) $(MAYBE_IDL) + endif +else + OBJS_LIBS := $(CLIBS_OPTS) $(MAYBE_IDL) +endif + +# If we have to make byte-code +ifndef REAL_OCAMLC + # EXTRADEPS is added dependencies we have to insert for all + # executable files we generate. Ideally it should be all of the + # libraries we use, but it's hard to find the ones that get searched on + # the path since I don't know the paths built into the compiler, so + # just include the ones with slashes in their names. + EXTRADEPS := $(addsuffix .cma,$(foreach i,$(LIBS),$(if $(findstring /,$(i)),$(i)))) + SPECIAL_OCAMLFLAGS := $(OCAMLBCFLAGS) + + REAL_OCAMLC := $(INTF_OCAMLC) + + REAL_IMPL := $(IMPL_CMO) + REAL_IMPL_INTF := $(IMPLO_INTF) + IMPL_SUF := .cmo + + DEPFLAGS := + MAKE_DEPS := $(MLDEPS) $(BCDEPIS) + + ifdef CREATE_LIB + CFLAGS := $(PIC_FLAGS) $(CFLAGS) + ifndef STATIC + ifneq ($(strip $(OBJ_LINK)),) + MAKEDLL := $(DLLSONAME) + ALL_LDFLAGS := -dllib $(DLLSONAME) + endif + endif + endif + + ifndef NO_CUSTOM + ifneq "$(strip $(OBJ_LINK) $(THREADS) $(MAYBE_IDL) $(CLIBS))" "" + ALL_LDFLAGS += -custom + endif + endif + + ALL_LDFLAGS += $(INCFLAGS) $(OCAMLLDFLAGS) $(OCAMLBLDFLAGS) \ + $(COMMON_LDFLAGS) $(LIBS:%=%.cma) + CAMLIDLDLLFLAGS := + + ifdef THREADS + ifdef VMTHREADS + THREAD_FLAG := -vmthread + else + THREAD_FLAG := -thread + endif + ALL_LDFLAGS := $(THREAD_FLAG) $(ALL_LDFLAGS) + ifndef CREATE_LIB + ifndef REAL_OCAMLFIND + ALL_LDFLAGS := unix.cma threads.cma $(ALL_LDFLAGS) + endif + endif + endif + +# we have to make native-code +else + EXTRADEPS := $(addsuffix .cmxa,$(foreach i,$(LIBS),$(if $(findstring /,$(i)),$(i)))) + ifndef PROFILING + SPECIAL_OCAMLFLAGS := $(OCAMLNCFLAGS) + PLDFLAGS := + else + SPECIAL_OCAMLFLAGS := -p $(OCAMLNCFLAGS) + PLDFLAGS := -p + endif + + REAL_IMPL := $(IMPL_CMX) + REAL_IMPL_INTF := $(IMPLX_INTF) + IMPL_SUF := .cmx + + CFLAGS := -DNATIVE_CODE $(CFLAGS) + + DEPFLAGS := -native + MAKE_DEPS := $(MLDEPS) $(NCDEPIS) + + ALL_LDFLAGS := $(PLDFLAGS) $(INCFLAGS) $(OCAMLLDFLAGS) \ + $(OCAMLNLDFLAGS) $(COMMON_LDFLAGS) + CAMLIDLDLLFLAGS := -opt + + ifndef CREATE_LIB + ALL_LDFLAGS += $(LIBS:%=%.cmxa) + else + CFLAGS := $(PIC_FLAGS) $(CFLAGS) + endif + + ifdef THREADS + THREAD_FLAG := -thread + ALL_LDFLAGS := $(THREAD_FLAG) $(ALL_LDFLAGS) + ifndef CREATE_LIB + ifndef REAL_OCAMLFIND + ALL_LDFLAGS := unix.cmxa threads.cmxa $(ALL_LDFLAGS) + endif + endif + endif +endif + +export MAKE_DEPS + +ifdef ANNOTATE + ANNOT_FLAG := -dtypes +else +endif + +ALL_OCAMLCFLAGS := $(THREAD_FLAG) $(ANNOT_FLAG) $(OCAMLFLAGS) \ + $(INCFLAGS) $(SPECIAL_OCAMLFLAGS) + +ifdef make_deps + -include $(MAKE_DEPS) + PRE_TARGETS := +endif + +########################################################################### +# USER RULES + +# Call "OCamlMakefile QUIET=" to get rid of all of the @'s. +QUIET=@ + +# generates byte-code (default) +byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes +bc: byte-code + +byte-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(BCRESULT)" make_deps=yes +bcnl: byte-code-nolink + +top: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(TOPRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes + +# generates native-code + +native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(NCRESULT) \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +nc: native-code + +native-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +ncnl: native-code-nolink + +# generates byte-code libraries +byte-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" \ + CREATE_LIB=yes \ + make_deps=yes +bcl: byte-code-library + +# generates native-code libraries +native-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).cmxa \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + CREATE_LIB=yes \ + make_deps=yes +ncl: native-code-library + +ifdef WIN32 +# generates byte-code dll +byte-code-dll: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).dll \ + REAL_RESULT="$(BCRESULT)" \ + make_deps=yes +bcd: byte-code-dll + +# generates native-code dll +native-code-dll: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).dll \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +ncd: native-code-dll +endif + +# generates byte-code with debugging information +debug-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dc: debug-code + +debug-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dcnl: debug-code-nolink + +# generates byte-code libraries with debugging information +debug-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + CREATE_LIB=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dcl: debug-code-library + +# generates byte-code for profiling +profiling-byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" PROFILING="y" \ + make_deps=yes +pbc: profiling-byte-code + +# generates native-code + +profiling-native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(NCRESULT) \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + PROFILING="y" \ + make_deps=yes +pnc: profiling-native-code + +# generates byte-code libraries +profiling-byte-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" PROFILING="y" \ + CREATE_LIB=yes \ + make_deps=yes +pbcl: profiling-byte-code-library + +# generates native-code libraries +profiling-native-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).cmxa \ + REAL_RESULT="$(NCRESULT)" PROFILING="y" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + CREATE_LIB=yes \ + make_deps=yes +pncl: profiling-native-code-library + +# packs byte-code objects +pack-byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT).cmo \ + REAL_RESULT="$(BCRESULT)" \ + PACK_LIB=yes make_deps=yes +pabc: pack-byte-code + +# packs native-code objects +pack-native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(NCRESULT).cmx $(NCRESULT).o \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + PACK_LIB=yes make_deps=yes +panc: pack-native-code + +# generates HTML-documentation +htdoc: doc/$(RESULT)/html + +# generates Latex-documentation +ladoc: doc/$(RESULT)/latex + +# generates PostScript-documentation +psdoc: doc/$(RESULT)/latex/doc.ps + +# generates PDF-documentation +pdfdoc: doc/$(RESULT)/latex/doc.pdf + +# generates all supported forms of documentation +doc: htdoc ladoc psdoc pdfdoc + +########################################################################### +# LOW LEVEL RULES + +$(REAL_RESULT): $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) \ + $(OCAML_FIND_PACKAGES) $(OCAML_FIND_LINKPKG) \ + $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@$(EXE) \ + $(REAL_IMPL) + +nolink: $(REAL_IMPL_INTF) $(OBJ_LINK) + +ifdef WIN32 +$(REAL_RESULT).dll: $(REAL_IMPL_INTF) $(OBJ_LINK) + $(CAMLIDLDLL) $(CAMLIDLDLLFLAGS) $(OBJ_LINK) $(CLIBS) \ + -o $@ $(REAL_IMPL) +endif + +%$(TOPSUFFIX): $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) + $(REAL_OCAMLFIND) $(OCAMLMKTOP) \ + $(OCAML_FIND_PACKAGES) $(OCAML_FIND_LINKPKG) \ + $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@$(EXE) \ + $(REAL_IMPL) + +.SUFFIXES: .mli .ml .cmi .cmo .cmx .cma .cmxa .$(EXT_OBJ) \ + .mly .di .d .$(EXT_LIB) .idl .c .$(EXT_CXX) .h .so \ + .rep .zog .glade +ifndef MSVC +$(DLLSONAME): $(OBJ_LINK) + $(OCAMLMKLIB) $(INCFLAGS) $(CLIBFLAGS) \ + -o $(CLIB_BASE) $(OBJ_LINK) $(CLIBS:%=-l%) \ + $(OCAMLMKLIB_FLAGS) +endif + +$(RESULT).cma: $(REAL_IMPL_INTF) $(MAKEDLL) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -a $(ALL_LDFLAGS) \ + $(OBJS_LIBS) -o $@ $(OCAMLBLDFLAGS) $(REAL_IMPL) + +$(RESULT).cmxa $(RESULT).$(EXT_LIB): $(REAL_IMPL_INTF) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(OCAMLOPT) -a $(ALL_LDFLAGS) $(OBJS_LIBS) \ + $(OCAMLNLDFLAGS) -o $@ $(REAL_IMPL) + +$(RES_CLIB): $(OBJ_LINK) +ifndef MSVC + ifneq ($(strip $(OBJ_LINK)),) + $(AR) rcs $@ $(OBJ_LINK) + endif +else + ifneq ($(strip $(OBJ_LINK)),) + lib -nologo -debugtype:cv -out:$(RES_CLIB) $(OBJ_LINK) + endif +endif + +.mli.cmi: $(EXTRADEPS) + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + echo $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + else \ + echo $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp \"$$pp $(PPFLAGS)\" $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp "$$pp $(PPFLAGS)" $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + fi + +.ml.cmi .ml.$(EXT_OBJ) .ml.cmx .ml.cmo: $(EXTRADEPS) + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + echo $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(ALL_OCAMLCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(ALL_OCAMLCFLAGS) $<; \ + else \ + echo $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp \"$$pp $(PPFLAGS)\" $(ALL_OCAMLCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp "$$pp $(PPFLAGS)" $(ALL_OCAMLCFLAGS) $<; \ + fi + +ifdef PACK_LIB +$(REAL_RESULT).cmo $(REAL_RESULT).cmx $(REAL_RESULT).o: $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -pack $(ALL_LDFLAGS) \ + $(OBJS_LIBS) -o $@ $(REAL_IMPL) +endif + +.PRECIOUS: %.ml +%.ml: %.mll + $(OCAMLLEX) $< + +.PRECIOUS: %.ml %.mli +%.ml %.mli: %.mly + $(OCAMLYACC) $(YFLAGS) $< + +.PRECIOUS: %.ml +%.ml : %.rep + $(CAMELEON_REPORT) $(CAMELEON_REPORT_FLAGS) -gen $< + +.PRECIOUS: %.ml +%.ml : %.zog + $(CAMELEON_ZOGGY) $(CAMELEON_ZOGGY_FLAGS) -impl $< > $@ + +.PRECIOUS: %.ml +%.ml : %.glade + $(OCAML_GLADECC) $(OCAML_GLADECC_FLAGS) $< > $@ + +.PRECIOUS: %.ml %.mli %_stubs.c %.h +%.ml %.mli %_stubs.c %.h: %.idl + $(CAMLIDL) $(MAYBE_IDL_HEADER) $(IDLFLAGS) \ + $(CAMLIDLFLAGS) $< + $(QUIET)if [ $(NOIDLHEADER) ]; then touch $*.h; fi + +.c.$(EXT_OBJ): + $(OCAMLC) -c -cc "$(CC) $(CFLAGS) $(CINCFLAGS)" \ + $< $(CFLAG_O)$@ + +.$(EXT_CXX).$(EXT_OBJ): + $(CXX) -c $(CXXFLAGS) $(CINCFLAGS) -I'$(OCAMLLIBPATH)' \ + $< $(CFLAG_O)$@ + +$(MLDEPDIR)/%.d: %.ml + $(QUIET)echo making $@ from $< + $(QUIET)if [ ! -d $(@D) ]; then mkdir -p $(@D); fi + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + $(DINCFLAGS) $< > $@; \ + else \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + -pp "$$pp $(PPFLAGS)" $(DINCFLAGS) $< > $@; \ + fi + +$(BCDIDIR)/%.di $(NCDIDIR)/%.di: %.mli + $(QUIET)echo making $@ from $< + $(QUIET)if [ ! -d $(@D) ]; then mkdir -p $(@D); fi + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) $(DINCFLAGS) $< > $@; \ + else \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) \ + -pp "$$pp $(PPFLAGS)" $(DINCFLAGS) $< > $@; \ + fi + +doc/$(RESULT)/html: $(DOC_FILES) + rm -rf $@ + mkdir -p $@ + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + echo $(OCAMLDOC) -html -d $@ $(OCAMLDOCFLAGS) $(INCFLAGS) $(DOC_FILES); \ + $(OCAMLDOC) -html -d $@ $(OCAMLDOCFLAGS) $(INCFLAGS) $(DOC_FILES); \ + else \ + echo $(OCAMLDOC) -pp \"$$pp $(PPFLAGS)\" -html -d $@ $(OCAMLDOCFLAGS) \ + $(INCFLAGS) $(DOC_FILES); \ + $(OCAMLDOC) -pp "$$pp $(PPFLAGS)" -html -d $@ $(OCAMLDOCFLAGS) \ + $(INCFLAGS) $(DOC_FILES); \ + fi + +doc/$(RESULT)/latex: $(DOC_FILES) + rm -rf $@ + mkdir -p $@ + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + echo $(OCAMLDOC) -latex $(OCAMLDOCFLAGS) $(INCFLAGS) \ + $(DOC_FILES) -o $@/doc.tex; \ + $(OCAMLDOC) -latex $(OCAMLDOCFLAGS) $(INCFLAGS) $(DOC_FILES) \ + -o $@/doc.tex; \ + else \ + echo $(OCAMLDOC) -pp \"$$pp $(PPFLAGS)\" -latex $(OCAMLDOCFLAGS) \ + $(INCFLAGS) $(DOC_FILES) -o $@/doc.tex; \ + $(OCAMLDOC) -pp "$$pp $(PPFLAGS)" -latex $(OCAMLDOCFLAGS) \ + $(INCFLAGS) $(DOC_FILES) -o $@/doc.tex; \ + fi + +doc/$(RESULT)/latex/doc.ps: doc/$(RESULT)/latex + cd doc/$(RESULT)/latex && \ + $(LATEX) doc.tex && \ + $(LATEX) doc.tex && \ + $(DVIPS) $(DVIPSFLAGS) doc.dvi -o $(@F) + +doc/$(RESULT)/latex/doc.pdf: doc/$(RESULT)/latex/doc.ps + cd doc/$(RESULT)/latex && $(PS2PDF) $( + let boothosts_cf = + if is_file "/usr/local/jumpstart/conf/admins.cf" then + "/usr/local/jumpstart/conf/admins.cf" + else + "/JumpStart/conf/admins.cf" in + let ch = open_in boothosts_cf in + let netblock_boothost = Hashtbl.create 253 in + let boothost_netblock = Hashtbl.create 253 in + let boothost_re = Pcre.regexp "^([-\\w.]+):\\s*$" and + netmask_re = Pcre.regexp "^\\s+-\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+/\\d+)\\s*$" and + cur_boothost = ref "" in + + let rec loop () = + let line = input_line ch in + if Pcre.pmatch ~rex:boothost_re line then + cur_boothost := (Pcre.extract ~rex:boothost_re ~full_match:false line).(0) + else + if Pcre.pmatch ~rex:netmask_re line then begin + let netmask = (Pcre.extract ~rex:netmask_re ~full_match:false line).(0) in + let net = (Netmask.netmask_of_string netmask) in + Hashtbl.add netblock_boothost net !cur_boothost; + Hashtbl.add boothost_netblock !cur_boothost net + end; + loop () in + let header = input_line ch in + if header <> "---" then + raise (AdminsErr ("Wrong header admins.cf: " ^ header)); + try + loop (); + with + End_of_file -> (netblock_boothost, boothost_netblock)) + +let boothosts_for_netblock netblock = + let hash, _ = boothost_file_memo.get () in + try + Some (Hashtbl.find_all hash (Netmask.netmask_of_string netblock)) + with Not_found -> + do_warning "NOBOOTHOST_NETBLOCK" netblock; + None + +let netblocks_for_boothost boothost = + let _, hash = boothost_file_memo.get () in + try + Some (List.map Netmask.string_of_netmask (Hashtbl.find_all hash boothost)) + with Not_found -> + do_warning "NONETBLOCK_BOOTHOST" boothost; + None + +let all_boothosts () = + let _, hash = boothost_file_memo.get () in + hash_keys hash + +let clear_cache () = boothost_file_memo.clear () diff --git a/librange/source/admins.mli b/librange/source/admins.mli new file mode 100644 index 0000000..e9b206b --- /dev/null +++ b/librange/source/admins.mli @@ -0,0 +1,4 @@ +val boothosts_for_netblock : string -> string list option +val netblocks_for_boothost : string -> string list option +val all_boothosts : unit -> string list +val clear_cache : unit -> unit diff --git a/librange/source/ast.mli b/librange/source/ast.mli new file mode 100644 index 0000000..57657b1 --- /dev/null +++ b/librange/source/ast.mli @@ -0,0 +1,19 @@ +type expr = + | EmptyExpr + | Literal of string + | Range of string * string * string * string + | Braces of expr * expr * expr + | Regex of string + | HostGroup of expr + | Parens of expr + | Union of expr * expr + | Diff of expr * expr + | Inter of expr * expr + | Cluster of expr * expr + | Admin of expr + | GetCluster of expr + | Filter of expr * string + | NotFilter of expr * string + | GetGroup of expr + | Function of string * string + diff --git a/librange/source/configure b/librange/source/configure new file mode 100755 index 0000000..039e4d0 --- /dev/null +++ b/librange/source/configure @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/librange/source/evaluate.ml b/librange/source/evaluate.ml new file mode 100644 index 0000000..152d8fd --- /dev/null +++ b/librange/source/evaluate.ml @@ -0,0 +1,714 @@ +open Memoize +open Printf +open Range_utils + +exception SyntaxErr of string +exception RangeErr of string +exception FuncErr of string + +module StrSet = Set.Make(String) + +let do_caching = ref true +let want_caching t = do_caching := t + +let use_ranged = ref false +let want_ranged t = use_ranged := t + +(* return a list of directories under this path with nodes.cf files *) +let get_cluster_dirs path = + let dh = Unix.opendir path in + let entries = ref [] in + try + while true do + let entry = (Unix.readdir dh) in + match entry.[0] with + | '.' -> () + | _ -> let fullname = path ^ "/" ^ entry in + if is_dir fullname && is_file (fullname ^ "/nodes.cf") + then entries := entry::!entries; + done; + [] + with End_of_file -> Unix.closedir dh; !entries;; + + +(* create a set from a list of strings *) +let set_of_list lst = + let rec set_of_list_rec lst acc = + match lst with + | [] -> acc + | hd::tl -> set_of_list_rec tl (StrSet.add hd acc) in + set_of_list_rec lst StrSet.empty;; + +let range_altpath = ref "" +let range_set_altpath s = range_altpath := s + +let file_exists f = try (Unix.stat f).Unix.st_kind = Unix.S_REG +with Unix.Unix_error _ -> false + +let get_cluster_file cluster = + let alt = !range_altpath in + let locations = [alt ^ "/" ^ cluster ^ "/tools/conf/nodes.cf"; + alt ^ "/" ^ cluster ^ "/nodes.cf"; + "/home/seco/tools/conf/" ^ cluster ^ "/nodes.cf"; + "/usr/local/gemclient/" ^ cluster ^ "/nodes.cf"] in + List.find file_exists locations + +let ignore_clusters_file () = + let alt = !range_altpath in + let locations = [ + alt ^ "/all/tools/conf/IGNORE"; + alt ^ "/all/IGNORE"; + "/home/seco/tools/conf/all/IGNORE"] in + List.find file_exists locations + +(* Dirs to ignore for whoismycluster *) +(* TODO Add a warning when IGNORE is not found *) +let ignore_dirs_memo = + memoize + (fun () -> + try + let mark_cluster hash cluster = Hashtbl.add hash cluster true in + let c = read_big_file (ignore_clusters_file()) in + let h = Hashtbl.create (List.length c) in + List.iter (mark_cluster h) c; + h + with Not_found -> Hashtbl.create 1) + +(* Should we consider this directory for whoismycluster *) +let interesting_dir dir = + let h = ignore_dirs_memo.get () in + not (Hashtbl.mem h dir);; + +let read_cluster_file cluster = + try + read_big_file (get_cluster_file cluster) + with Not_found -> ( + do_warning "NOCLUSTERDEF" cluster; + [""] + ) + +(* get the location of the vips.cf file *) +let get_vips_file cluster = + let alt = !range_altpath in + let locations = [ alt ^ "/" ^ cluster ^ "/tools/conf/vips.cf"; + alt ^ "/" ^ cluster ^ "/vips.cf"; + "/home/seco/tools/conf/" ^ cluster ^ "/vips.cf"] in + List.find file_exists locations + +(* create a set with the vips for a given cluster *) +let get_cluster_vips_field cluster field = + try + set_of_list (List.map (fun ln -> List.nth (Pcre.split ln) field) + (good_lines (read_big_file (get_vips_file cluster)))) + with + Not_found -> StrSet.empty + +(* create a set with the vips for a given cluster *) +let get_cluster_vips cluster = + get_cluster_vips_field cluster 0 + +let get_cluster_viphosts cluster = + get_cluster_vips_field cluster 1 + +(* utility function to get the key of a hash w/o raising Not_found *) +let find_key h key = + try Hashtbl.find h key + with Not_found -> "" + +(* how deep are we recursing *) +let eval_recursion = ref 0 + +(* Construct a set for the given range *) +let range prefix start finish domain = + let len = String.length start + and len_end = String.length finish in + let padded_end = + if len > len_end then + (String.sub start 0 (len - len_end)) ^ finish + else finish in + let istart = int_of_string(start) and + iend = int_of_string(padded_end) in + let fmt_node pre n suf = + pre ^ (pad0 n len) ^ suf in + let rec add_elts set i = + if i < iend then + add_elts (StrSet.add (fmt_node prefix i domain) set) (i+1) + else + StrSet.add (fmt_node prefix iend domain) set in + if istart < iend then + add_elts StrSet.empty istart + else if istart = iend then + StrSet.singleton (prefix ^ start ^ domain) + else + StrSet.singleton (prefix ^ start ^ "-" ^ finish ^ domain) + +let union a b = StrSet.union a b +let diff a b = StrSet.diff a b +let inter a b = StrSet.inter a b +let literal str = StrSet.singleton str +let elements a = StrSet.elements a +(* braces generates all the combinations of pre_set ^ mid_set ^ post_set *) +let braces pre_set_arg mid_set_arg post_set_arg = + (* return a set with a single element 'default' if the given set is empty *) + let empty_set set default = match (StrSet.cardinal set) with + | 0 -> StrSet.singleton default + | _ -> set in + let pre_set = empty_set pre_set_arg "" and + mid_set = empty_set mid_set_arg "" and + post_set = empty_set post_set_arg "" in + StrSet.fold + (fun post a_post -> + union + (StrSet.fold + (fun pre a_pre -> + union + (StrSet.fold + (fun x a_mid -> StrSet.add (pre ^ x ^ post) a_mid) + mid_set StrSet.empty) + a_pre) + pre_set StrSet.empty) + a_post) + post_set StrSet.empty + +let rec optimize expr = + match expr with + (* Optimizations here *) + | Ast.Inter(e, Ast.Regex re) -> Ast.Filter ((optimize e), re) + | Ast.Diff(e, Ast.Regex re) -> Ast.NotFilter ((optimize e), re) +(* don't know how to optimize these so we just copy them *) + | Ast.Braces(pre, e, post) -> Ast.Braces (pre, (optimize e), post) + | Ast.HostGroup hg -> Ast.HostGroup (optimize hg) + | Ast.Parens expr -> Ast.Parens (optimize expr) + | Ast.Union (a, b) -> Ast.Union ((optimize a),(optimize b)) + | Ast.Diff (a, b) -> Ast.Diff ((optimize a),(optimize b)) + | Ast.Inter (a, b) -> Ast.Inter ((optimize a),(optimize b)) + | Ast.Cluster (c, k) -> Ast.Cluster ((optimize c),(optimize k)) + | Ast.Admin nodes -> Ast.Admin (optimize nodes) + | Ast.GetCluster nodes -> Ast.GetCluster (optimize nodes) + | Ast.Filter (nodes, re) -> Ast.Filter ((optimize nodes), re) + | Ast.NotFilter (nodes, re) -> Ast.NotFilter ((optimize nodes), re) + | Ast.GetGroup nodes -> Ast.GetGroup (optimize nodes) + | _ -> expr;; + +let rec evaluate expr = + match expr with + | Ast.EmptyExpr -> StrSet.empty + | Ast.Literal s -> StrSet.singleton s + | Ast.Range (prefix, start, finish, domain) + -> range prefix start finish domain + | Ast.Braces (pre, elts, post) + -> braces (evaluate pre) (evaluate elts) (evaluate post) + | Ast.Regex re + -> regex re + | Ast.HostGroup hg -> hostsgroups_of (evaluate hg) + | Ast.Parens expr -> evaluate expr + | Ast.Union (a, b) -> union (evaluate a) (evaluate b) + | Ast.Diff (a, b) -> diff (evaluate a) (evaluate b) + | Ast.Inter (a, b) -> inter (evaluate a) (evaluate b) + | Ast.Cluster (clusters, keys) -> exp_clusters (evaluate clusters) (evaluate keys) + | Ast.Admin nodes -> admin_of (evaluate nodes) + | Ast.GetCluster nodes -> fast_cluster_of (evaluate nodes) + | Ast.Filter (nodes, re) -> filter (evaluate nodes) re + | Ast.NotFilter (nodes, re) -> notfilter (evaluate nodes) re + | Ast.GetGroup nodes -> groups_of (evaluate nodes) + | Ast.Function (name, args) -> eval_func name args +and expr_of_string s = + let lexbuf = Lexing.from_string s in + Parser.main Lexer.token lexbuf +and eval_string_not_memo s = + match String.length s with + | 0 -> StrSet.empty + | _ -> + try evaluate (optimize (expr_of_string s)) + with Parsing.Parse_error -> + raise (SyntaxErr (sprintf "Syntax error: [%s]" s)); +and eval_memo = Hashtbl.create 1023 +and clear_eval_cache () = Hashtbl.clear eval_memo +and eval_string s = + try + Hashtbl.find eval_memo s + with Not_found -> + incr eval_recursion; + if !eval_recursion > 32 then + raise (RangeErr("Recursion Too Deep: " ^ s)); + let res = eval_string_not_memo s in + Hashtbl.add eval_memo s res ; + decr eval_recursion; + res +and cluster_keys_memo = Hashtbl.create 511 +and clear_cluster_keys_cache () = Hashtbl.clear cluster_keys_memo +and cluster_keys_not_memo cluster = + let cur_range = ref [] and + cur_key = ref "" and + h : (string, string) Hashtbl.t = Hashtbl.create 19 in + let parsed_cur_range () = + let key_re = Pcre.regexp "\\$(\\w+)" and + cluster_subst = Pcre.subst ("%" ^ cluster ^ ":$1") and + r = String.concat "," (List.rev !cur_range) in + Pcre.replace ~rex:key_re ~itempl:cluster_subst r in + let add_cur_key hash = + if String.length !cur_key > 0 + then Hashtbl.add hash !cur_key (parsed_cur_range ()) + else () in + let inc_re = Pcre.regexp "^\\s+INCLUDE\\s+" and + exc_re = Pcre.regexp "^\\s+EXCLUDE\\s+" in + let include_line ln = Pcre.pmatch ~rex:inc_re ln and + exclude_line ln = Pcre.pmatch ~rex:exc_re ln in + let get_include_line ln = Pcre.replace ~rex:inc_re ln and + get_exclude_line ln = "-(" ^ (Pcre.replace ~rex:exc_re ln) ^ ")" in + let process_line line = + let c = line.[0] in + if c = ' ' || c = '\t' then + if include_line line then + cur_range := (get_include_line line)::!cur_range + else + if exclude_line line then + cur_range := (get_exclude_line line)::!cur_range + else + do_warning "SYNTAXERROR" (cluster ^ ": [" ^ line ^ "]") + else begin + add_cur_key h; + cur_key := line; + cur_range := []; + end in + + List.iter process_line (good_lines (read_cluster_file cluster)); + add_cur_key h; + h +and cluster_keys cluster = + try + Hashtbl.find cluster_keys_memo cluster + with Not_found -> + let keys = cluster_keys_not_memo cluster in + Hashtbl.add cluster_keys_memo cluster keys ; + keys +and expand_one_cluster cluster key = + match key with + | "VIPS" -> get_cluster_vips cluster + | "VIPHOSTS" -> get_cluster_viphosts cluster + | _ -> let ckeys = cluster_keys(cluster) in + if key = "KEYS" then set_of_list (hash_keys ckeys) + else eval_string (match key with + | "DOWN" -> "%" ^ cluster ^ ":ALL,-%" ^ cluster ^ ":CLUSTER" + | "UP" -> "%" ^ cluster ^ ":CLUSTER" + | _ -> find_key ckeys key) +and exp_clusters cls keys = + let exp_cl_keys c = + StrSet.fold (fun k ka -> union (expand_one_cluster c k) ka) + keys StrSet.empty in + StrSet.fold (fun c ca -> union (exp_cl_keys c) ca) cls StrSet.empty +and all_nodes () = expand_one_cluster "GROUPS" "ALL" +and filter nodes re = + let rex=Pcre.regexp re in + StrSet.filter (fun x -> Pcre.pmatch ~rex x) nodes +and notfilter nodes re = + let rex=Pcre.regexp re in + StrSet.filter (fun x -> not (Pcre.pmatch ~rex x)) nodes +and regex re = filter (all_nodes ()) re +and hostsgroups_of hg = + StrSet.fold (fun x a -> union ( + match x.[0] with + 'a'..'z' -> expand_one_cluster "HOSTS" x + | _ -> expand_one_cluster "GROUPS" x + ) a) hg + StrSet.empty +and admins_hash_memo = ref None +and clear_admins_hash_cache () = admins_hash_memo := None +and admins_hash () = match !admins_hash_memo with + None -> + let add_node h a n = Hashtbl.add h n a in + let add_nodes_from_admin h a = + let nodes = expand_one_cluster "HOSTS" a in + List.iter (add_node h a) (StrSet.elements nodes) in + let h = Hashtbl.create 21073 in + let admins = expand_one_cluster "HOSTS" "KEYS" in + StrSet.iter (add_nodes_from_admin h) admins; + admins_hash_memo := Some h; + h + | Some hash -> hash +and admin_of nodes = + let admin_for node = + let h = admins_hash () in + try Some (Hashtbl.find h node) + with Not_found -> + do_warning "NOADMIN" node; + None in + let rec admins node_lst acc = + match node_lst with + | [] -> acc + | hd::tl -> match (admin_for hd) with + Some a -> admins tl (StrSet.add a acc) + | None -> admins tl acc in + admins (StrSet.elements nodes) StrSet.empty +and cluster_hash_memo = ref None +and clear_cluster_hash_cache () = cluster_hash_memo := None +and cluster_hash lst = match !cluster_hash_memo with + None -> + let add_cluster h cl = + let add_node h n = Hashtbl.add h n cl in + let cl_nodes = expand_one_cluster cl "ALL" in + StrSet.iter (add_node h) cl_nodes in + let h = Hashtbl.create 41073 in + List.iter (add_cluster h) lst; + cluster_hash_memo := Some h; + h + | Some hash -> hash +(* TODO use alt_path *) +and all_clusters = + memoize (fun () -> + let dir = "/home/seco/tools/conf" in + List.filter interesting_dir (get_cluster_dirs dir)) +and get_all_clusters nodes = + let cl = all_clusters.get() in + let all_cluster_for node = + let node_cl_hash = cluster_hash(cl) in + try + Some (Hashtbl.find_all node_cl_hash node) + with Not_found -> + do_warning "NOCLUSTER" node; + None in + let rec clusters node_lst acc = + match node_lst with + | [] -> acc + | node :: tl -> match (all_cluster_for node) with + | None -> clusters tl acc + | Some c -> clusters tl (StrSet.union (set_of_list c) acc) in + clusters (StrSet.elements nodes) StrSet.empty +and fast_cluster_of nodes = + let cl = all_clusters.get() in + let node_cl_hash = cluster_hash(cl) in + let best_guess_re = Pcre.regexp "^([a-z]+\\d\\d\\d)" in + let fast_cluster_for node = + try + let res = Pcre.extract ~rex:best_guess_re ~full_match:false node in + let potential_cluster = res.(0) in + if Hashtbl.mem node_cl_hash potential_cluster then ( + let prev_warnings = Range_utils.want_warnings false in + let cl_nodes = expand_one_cluster potential_cluster "ALL" in + ignore (Range_utils.want_warnings prev_warnings); + if StrSet.mem node cl_nodes then + Some potential_cluster + else None) + else + None + with Not_found -> None in + let slow_cluster_for node = + try + Some (Hashtbl.find node_cl_hash node) + with Not_found -> + do_warning "NOCLUSTER" node; + None in + let rec clusters node_lst acc = + match node_lst with + | [] -> acc + | node :: tl -> match (fast_cluster_for node) with + | None -> + (match (slow_cluster_for node) with + | None -> clusters tl acc + | Some c -> clusters tl (StrSet.add c acc)) + | Some c -> clusters tl (StrSet.add c acc) in + clusters (StrSet.elements nodes) StrSet.empty +and eval_func name args = + match name with + | "mem" -> + begin + let comma = Pcre.regexp "\\s*[,;]\\s*" in + let arg_list = Pcre.split ~rex:comma args in + match arg_list with + cluster :: rest -> + let range = (String.concat "," rest) in + member_of cluster (eval_string range) + | _ -> raise (FuncErr("Invalid arguments to #mem: " ^ args)) + end + | "boot_v" -> boothost_for_vlan (eval_string args) + | "vlan" -> netblocks (eval_string args) + | "bh" -> eval_string ("boot_v(vlan(" ^ args ^ "))") (* just a shorthand *) + | "dc" -> colos (eval_string args) + | "clusters" -> get_all_clusters (eval_string args) + | "allclusters" -> list_all_clusters () + | "v_dc" | "vlans_dc" -> vlans_for_dc (eval_string args) + | "ip" -> resolve_ips (eval_string args) + | "v_boot" -> vlans_for_boothost (eval_string args) + | "hosts_v" -> hosts_in_vlan (eval_string args) + | "hosts_dc" -> hosts_in_dc (eval_string args) + | "boot" | "boothosts" -> all_boothosts () + | "q" -> StrSet.singleton args + | "has" -> + begin + try + let rex = Pcre.regexp "(.+?)[,;]\\s*(.+)" in + let res = Pcre.extract ~full_match:false ~rex args in + let lbl = res.(0) and range = res.(1) in + has_label lbl (eval_string range) + with Not_found -> raise (FuncErr("Invalid arguments to #has: " ^ args)) + end + | "count" -> StrSet.singleton (string_of_int (StrSet.cardinal (eval_string args))) + | "lim" | "limit" -> + begin + try + let comma = Pcre.regexp "^(\\d+)[,;](.+)$" in + let arg_ary = Pcre.extract ~full_match:false ~rex:comma args in + let n = arg_ary.(0) in + let range = arg_ary.(1) in + let int_n = int_of_string n in + let res_set = eval_string range in + let elts = ref 0 in + let limit_set = + StrSet.filter (fun x -> (incr elts; !elts <= int_n)) res_set in + limit_set + with Not_found -> raise (FuncErr("Invalid arguments to #lim: " ^ args)) + end + | _ as x -> raise (FuncErr("Invalid function " ^ x)) +and member_of cluster nodes = + let node_elements = StrSet.elements nodes in + let groups = expand_one_cluster cluster "KEYS" in + let any_node_in this_set = + let rec node_in node_lst = + match node_lst with + | [] -> false + | hd::tl -> if StrSet.mem hd this_set then + true + else + node_in tl in + node_in node_elements in + let rec g_n group_lst acc = + match group_lst with + | [] -> acc + | hd::tl -> + let group = expand_one_cluster cluster hd in + if any_node_in group then + g_n tl (StrSet.add hd acc) + else + g_n tl acc in + g_n (StrSet.elements groups) StrSet.empty +and groups_of nodes = + member_of "GROUPS" nodes +and resolve_ips nodes = + StrSet.fold ( + fun node acc -> match (Tinydns.get_ip node) with + | None -> acc + | Some ip -> StrSet.add (Tinydns.string_of_ip ip) acc + ) nodes StrSet.empty +and netblocks nodes = + StrSet.fold ( + fun node acc -> + match (Hosts_netblocks.netblock_for_host node) with + | None -> + (match (Tinydns.get_ip node) with + | None -> acc + | Some ip -> + match (Netblocks.find_netblock ip) with + | None -> acc + | Some (net, _) -> StrSet.add net acc) + | Some net -> StrSet.add net acc) + nodes StrSet.empty +and colos nodes = + StrSet.fold ( + fun node acc -> + match (Tinydns.get_ip node) with + | None -> acc + | Some ip -> + match (Netblocks.find_netblock ip) with + None -> acc + | Some (_, colo) -> StrSet.add colo acc + ) nodes StrSet.empty +and vlans_for_boothost boothosts = + StrSet.fold ( + fun boothost acc -> + match (Admins.netblocks_for_boothost boothost) with + | None -> acc + | Some blocks -> StrSet.union acc (set_of_list blocks) + ) boothosts StrSet.empty +and all_boothosts () = set_of_list (Admins.all_boothosts()) +and boothost_for_vlan vlans = + StrSet.fold ( + fun vlan acc -> + match (Admins.boothosts_for_netblock vlan) with + | None -> acc + | Some boothosts -> StrSet.union acc (set_of_list boothosts) + ) vlans StrSet.empty +and hosts_in_vlan vlans = + StrSet.fold (fun vlan acc -> match Hosts_netblocks.hosts_in_netblock vlan with + None -> acc + | Some lst -> StrSet.union acc (set_of_list lst)) + vlans StrSet.empty +and vlans_for_dc dcs = + StrSet.fold ( + fun dc acc -> + let netblocks = set_of_list (Netblocks.vlans_for_dc dc) in + StrSet.union acc netblocks) dcs StrSet.empty +and hosts_in_dc dcs = + StrSet.fold ( + fun dc acc -> + let netblocks_dc = set_of_list (Netblocks.vlans_for_dc dc) in + StrSet.union acc (hosts_in_vlan netblocks_dc)) dcs StrSet.empty +and has_label lbl elts = + (* pick a random element - hopefully it should be only one *) + let matches_label cluster = StrSet.subset elts (expand_one_cluster cluster lbl) in + let cluster_lst = all_clusters.get() in + StrSet.filter matches_label (set_of_list cluster_lst) + +and list_all_clusters () = + set_of_list (all_clusters.get()) + +let node_regex = Pcre.regexp ( + "^([-\\w.]*?)" (* prefix *) + ^ "(\\d+)" (* followed by the node number *) + ^ "(\\.[-A-Za-z\\d.]*[-A-Za-z]+[-A-Za-z\\d.]*)?" (* optional domain *) + ^ "$" (* end of string *) +) + +let sort_nodes a = + let n_to_tuple n = + try + let ary = Pcre.extract ~full_match:false ~rex:node_regex n in + ary.(0), ary.(2), int_of_string(ary.(1)), n + with Not_found -> "", "", 0, n in + let mapped = Array.map n_to_tuple a in + Array.fast_sort compare mapped ; + Array.map (fun tup -> match tup with _,_,_,n -> n) mapped + +(* Remove duplicates from a sorted string array *) +let unique nodes = + let seen = ref None and + result_lst = ref [] in + Array.iter + (fun x -> + ( match !seen with + | None -> result_lst := x :: !result_lst + | Some node when Some x <> !seen -> result_lst := x :: !result_lst + | _ -> (* Ignore if it's the = to the prev value *) () ); + seen := Some x) nodes; + Array.of_list (List.rev !result_lst) + +let compress_range nodes separator = + let ignore_common_prefix num1 num2 = + let str1 = string_of_int num1 and + str2 = string_of_int num2 in + let n = ref 0 and + l1 = String.length str1 and + l2 = String.length str2 in + if l1 < l2 then + str2 + else let min_len = min l1 l2 in + while (!n < min_len && str1.[!n] = str2.[!n]) do + incr n + done; + String.sub str2 !n (l2 - !n) in + + (* Converts a nodename to an array that has + (prefix) (domain) (nodenumber as string) (nodenumber as integer) *) + let n_to_tuple n = + try + let ary = Pcre.extract ~full_match:false ~rex:node_regex n in + ary.(0), ary.(2), ary.(1), int_of_string(ary.(1)) + with Not_found -> n, "", "", -2 in + + (* the previously seen prefix, node as int, node as str, domain, count + groups is a list of the components *) + let prev_pre = ref "" and + prev_inum = ref (-2) and + prev_num = ref "" and + prev_suf = ref "" and + prev_n = ref "" and + count = ref 0 and + groups = ref [] in + + (* add another component (like node) to our list of results *) + let add_one () = + groups := !prev_n :: !groups in + + (* add a group (like node1-10) to our list of results *) + let add_group () = + let b = Buffer.create 256 in + Buffer.add_string b !prev_pre; + Buffer.add_string b !prev_num; + Buffer.add_char b '-'; + Buffer.add_string b (ignore_common_prefix !prev_inum (!prev_inum + !count)); + Buffer.add_string b !prev_suf; + groups := (Buffer.contents b)::!groups in + + (* process a node *) + let do_node n = + let t = n_to_tuple n in + match t with + pre, suf, num, inum -> + if (pre = !prev_pre) && (suf = !prev_suf) && + (inum = (!prev_inum + !count + 1)) then + incr count + else begin + if String.length !prev_n > 0 then + if !count > 0 then + add_group () + else + add_one (); + prev_n := n; + prev_pre := pre; + prev_num := num; + prev_inum := inum; + prev_suf := suf; + count := 0 + end in + match Array.length nodes with + | 0 -> "" + | _ -> + Array.iter do_node (unique (sort_nodes nodes)); + (* Add the last group/node to our results *) + if !count > 0 then + add_group () + else + add_one (); + (* Return a comma separated list of nodes *) + String.concat separator (List.rev !groups) + +let compress_set set = + compress_range (Array.of_list (StrSet.elements set)) + +(* note that most of the caches are manually implemented as hashes + because for some reason ocaml refuses to accept memoize in the RHS of + let rec ... and ... - probably too complex to check for types +*) +let clear_caches () = + clear_eval_cache (); + clear_cluster_keys_cache (); + clear_admins_hash_cache (); + clear_cluster_hash_cache (); + ignore_dirs_memo.clear (); + all_clusters.clear (); + Admins.clear_cache (); + Netblocks.clear_cache (); + Tinydns.clear_cache () + +let debug_out lvl msg = + if lvl > 0 then (eprintf "librange: %s\n" msg; flush stderr) + +let expand_range str = + let runparam = + try + int_of_string (Sys.getenv "LIBRANGE_DEBUG") + with _ -> 0 in + let start_time = ref 0.0 in + if runparam > 1 then + start_time := Unix.gettimeofday(); + eval_recursion := 0 ; + if not !do_caching then ( + debug_out runparam "Clearing caches"; + clear_caches () ; + ); + let res = Array.of_list (StrSet.elements (eval_string str)) in + if runparam > 1 then ( + let elapsed_time = Unix.gettimeofday() -. !start_time in + if elapsed_time > 0.1 then ( + debug_out runparam + (sprintf "expand_range(%s) COMPLETED in %.2f" str elapsed_time); + if runparam > 2 then ( + Gc.print_stat stderr; + flush stderr))); + res + +let sorted_expand_range str = + let array = expand_range str in + sort_nodes array + +let want_warnings = Range_utils.want_warnings +let set_warning_callback = Range_utils.set_warning_callback diff --git a/librange/source/evaluate.mli b/librange/source/evaluate.mli new file mode 100644 index 0000000..28699ae --- /dev/null +++ b/librange/source/evaluate.mli @@ -0,0 +1,9 @@ +(* Public interface for the range parser *) +val range_set_altpath : string -> unit +val expand_range : string -> string array +val sorted_expand_range : string -> string array +val compress_range : string array -> string -> string +val want_caching : bool -> unit +val want_warnings : bool -> bool +val set_warning_callback : (string -> string -> unit) -> unit +val clear_caches : unit -> unit diff --git a/librange/source/hosts_netblocks.ml b/librange/source/hosts_netblocks.ml new file mode 100644 index 0000000..00cf8a2 --- /dev/null +++ b/librange/source/hosts_netblocks.ml @@ -0,0 +1,30 @@ +open Memoize;; + +let hosts_vlan_hash = + memoize + (fun () -> + let h_v = Hashtbl.create 41113 in + let v_h = Hashtbl.create 1097 in + let ip_hosts = Tinydns.all_ip_hosts () in + List.iter (fun (ip,host) -> match Netblocks.find_netblock ip with + None -> ignore () + | Some (net, _) -> + Hashtbl.add h_v host net; + Hashtbl.add v_h net host) ip_hosts; + (h_v, v_h)) + + +let hosts_in_netblock blk = + let _, v_h = hosts_vlan_hash.get () in + try + Some (Hashtbl.find_all v_h blk) + with Not_found -> None + +let netblock_for_host host = + let h_v, _ = hosts_vlan_hash.get () in + try + Some (Hashtbl.find h_v host) + with Not_found -> None + + + diff --git a/librange/source/lexer.mll b/librange/source/lexer.mll new file mode 100644 index 0000000..4049369 --- /dev/null +++ b/librange/source/lexer.mll @@ -0,0 +1,81 @@ +{ + open Parser + open Printf + open Pcre + open Range_utils + let re_buf = Buffer.create 256 + let args_buf = Buffer.create 1024 + (* optional prefix + digits + optional domain dash same prefix (optional) + + * digits + same optional domain if there was one, or an optional domain *) + let node_regex = regexp + "^([-\\w.]*?)(\\d+)(\\.[-A-Za-z\\d.]*[-A-Za-z]+[-A-Za-z\\d.]*)?-\\1?(\\d+)((?(3)\\3|(?:\\.[-A-Za-z\\d.]+)?))$" +} + +let digit = ['0'-'9'] +let letter = ['a'-'z' 'A'-'Z'] +let hostchar = letter | digit | '_' | '.' (* no dashes *) +let hostchard = hostchar | '-' + +rule token = parse + | [' ' '\t' '\n'] { token lexbuf } + | '#'? ((letter | '_')+ as name) '(' { FUNCTION (name, (args 1 lexbuf)) } + + (* optimization: don't use pcre unless the expression has a '-' *) + | (hostchar+ as literal) { LITERAL(literal) } + | (hostchar hostchard* as range) { + try + let res = extract ~full_match:false ~rex:node_regex range in + let pref = res.(0) and fst = res.(1) and dom = res.(2) and + lst = res.(3) and dom' = res.(4) + in RANGE(pref, fst, lst, (if dom = "" then dom' else dom)) + with Not_found -> LITERAL(range) } + | ",-" { DIFF } (* to keep people happy *) + | ",&" { INTER } (* to keep people happy *) + | '&' { INTER_HP } (* Higher precedence intersection *) + | ',' { UNION } + | '(' { LPAREN } + | ')' { RPAREN } + | '{' { LBRACE } + | '}' { RBRACE } + | '/' { REGEX (in_regex lexbuf) } + | ':' { COLON } + | '^' { ADMIN } + | '%' { CLUSTER_EXP } + | '@' { HOSTGROUP } + | '*' { GET_CLUSTER } + | '!' { NOT } + | '?' { GET_GROUP } + | hostchar+ as x { LITERAL (x) } + | eof { EOL } + | "-" { DIFF } (* this might break things *) + | _ as c { do_warning "LEXER" (Printf.sprintf "Invalid character: '%c'" c); token lexbuf } +and in_regex = parse + (* End of the regex *) + | '/' { let r = Buffer.contents re_buf in + Buffer.clear re_buf; + r } + | '\\' ( '\\' | '/' as c) { + Buffer.add_char re_buf c; + in_regex lexbuf } + | _ as c { + Buffer.add_char re_buf c; + in_regex lexbuf } + | eof { do_warning "LEXER" "Need end of regex delimiter"; "INVALID_REGEX" } +and args n = parse + | '(' { + Buffer.add_char args_buf '('; + args (n + 1) lexbuf } + | ')' { + Buffer.add_char args_buf ')'; + match n with + 0 -> do_warning "LEXER" "Need open parens first"; "INVALID_ARGS" + | 1 -> + let r = Buffer.contents args_buf in + Buffer.clear args_buf; + String.sub r 0 ((String.length r) - 1) (* get rid of the parens *) + | _ -> args (n - 1) lexbuf } + | _ as c { + Buffer.add_char args_buf c; + args n lexbuf } + | eof { do_warning "LEXER" "Need closing parens"; "INVALID_ARGS" } + diff --git a/librange/source/librange.c b/librange/source/librange.c new file mode 100644 index 0000000..37c5eda --- /dev/null +++ b/librange/source/librange.c @@ -0,0 +1,179 @@ + +#include +#include +#include +#include +#include + +#include + +#include "range.h" + +value *cb_range_parse; +value *cb_range_expand; +value *cb_range_compress; +value *cb_range_expand_sorted; +value *cb_range_set_altpath; +value *cb_range_clear_caches; +value *cb_range_want_caching; +value *cb_range_want_warnings; +value *cb_range_set_warning_callback; + + +/* holds exception, if any, last call threw */ + +char * ocaml_exception = NULL; + +/* Must be called before any range functions */ +void range_startup() { + char *argv[] = { "librange", NULL }; + caml_startup(argv); + cb_range_parse = caml_named_value("ocaml_parse_range"); + cb_range_expand = caml_named_value("ocaml_expand_range"); + cb_range_compress = caml_named_value("ocaml_compress_range"); + cb_range_expand_sorted = caml_named_value("ocaml_sorted_expand_range"); + cb_range_set_altpath = caml_named_value("ocaml_range_set_altpath"); + cb_range_clear_caches = caml_named_value("ocaml_clear_caches"); + cb_range_want_caching = caml_named_value("ocaml_want_caching"); + cb_range_want_warnings = caml_named_value("ocaml_want_warnings"); + cb_range_set_warning_callback = caml_named_value("ocaml_set_warning_callback"); +} + +int range_set_exception(value caml_result) { + if (Is_exception_result(caml_result)) { + if (ocaml_exception) free(ocaml_exception); + ocaml_exception = strdup(String_val(Field(Extract_exception(caml_result), 1))); + return 1; + } else { + range_clear_exception(); + return 0; + } +} + +void range_clear_exception() { + if (ocaml_exception) free(ocaml_exception); + ocaml_exception = NULL; +} +char * range_get_exception() { + return ocaml_exception; +} + +char * range_get_version() { + return LIBRANGE_VERSION; +} + +/* fixme no error checking */ +void range_free_nodes(const char ** nodes) { + int i; + for (i = 0; nodes[i] != NULL; i++) { + free(nodes[i]); /* free char pointers */ + } + free(nodes); /* free char* pointers */ +} + +/* Generic range expander */ + +const char ** meta_range_expand(value *cb, const char * c_range) { + const char ** c_result; + int i; + int num_nodes; + CAMLparam0(); + CAMLlocal2(caml_result, caml_range); + + caml_range = caml_copy_string(c_range); + caml_result = callback_exn(*cb, caml_range); + + if (range_set_exception(caml_result)) { + CAMLreturn(NULL); + } else { + num_nodes = Wosize_val(caml_result); + c_result = malloc( (num_nodes + 1) * sizeof(char*) ); + + for (i = 0; i < num_nodes; i++) { + char * t; + t = String_val(Field(caml_result,i)); + c_result[i] = strdup(t); + } + c_result[num_nodes] = NULL; + CAMLreturn(c_result); + } +} + +const char ** range_expand(const char * c_range) { + return meta_range_expand(cb_range_expand, c_range); +} +const char ** range_expand_sorted(const char * c_range) { + return meta_range_expand(cb_range_expand_sorted, c_range); +} + +/* + * Compress a bunch of nodes into a range + */ + +char * range_compress(const char ** c_nodes, const char* c_separator) { + CAMLparam0(); + CAMLlocal3(caml_result, caml_nodes, caml_separator); + + caml_nodes = copy_string_array(c_nodes); + caml_separator = caml_copy_string(c_separator); + caml_result = callback2_exn(*cb_range_compress, caml_nodes, caml_separator); + + if (range_set_exception(caml_result)) + CAMLreturn(NULL); + else + CAMLreturn(strdup(String_val(caml_result))); +} + +/* + * Parse a range by expanding, then compressing + */ + +char * range_parse(const char * c_range) { + CAMLparam0(); + CAMLlocal2(caml_result, caml_range); + + caml_range = caml_copy_string(c_range); + caml_result = callback_exn(*cb_range_parse, caml_range); + + if (range_set_exception(caml_result)) + CAMLreturn(NULL); + else + CAMLreturn(strdup(String_val(caml_result))); +} + +/* set altpath */ + +void range_set_altpath(const char * c_path) { + CAMLparam0(); + CAMLlocal2(caml_result, caml_path); + + caml_path = caml_copy_string(c_path); + caml_result = callback_exn(*cb_range_set_altpath, caml_path); + + range_set_exception(caml_result); + CAMLreturn0; +} + +void range_clear_caches() { + CAMLparam0(); + CAMLlocal1(caml_result); + caml_result = callback_exn(*cb_range_clear_caches, Val_unit); + range_set_exception(caml_result); + CAMLreturn0; +} + +void range_want_caching(int c_want) { + CAMLparam0(); + CAMLlocal1(caml_result); + caml_result = callback_exn(*cb_range_want_caching, Val_bool(c_want)); + range_set_exception(caml_result); + CAMLreturn0; +} + +void range_want_warnings(int c_want) { + CAMLparam0(); + CAMLlocal1(caml_result); + caml_result = callback_exn(*cb_range_want_warnings, Val_bool(c_want)); + range_set_exception(caml_result); + CAMLreturn0; +} diff --git a/librange/source/memoize.ml b/librange/source/memoize.ml new file mode 100644 index 0000000..f2e1191 --- /dev/null +++ b/librange/source/memoize.ml @@ -0,0 +1,13 @@ + +type ('a, 'b) memo = { get : ('a -> 'b) ; clear : (unit -> unit) } + +let memoize f = + let cache = Hashtbl.create 19 in + let clear_cache () = Hashtbl.clear cache in + let f' n = + try Hashtbl.find cache n + with Not_found -> + let fn = (f n) in + Hashtbl.add cache n fn; + fn in + { get = f' ; clear = clear_cache } diff --git a/librange/source/mlrange.ml b/librange/source/mlrange.ml new file mode 100644 index 0000000..f3d5369 --- /dev/null +++ b/librange/source/mlrange.ml @@ -0,0 +1,55 @@ +open Evaluate + +let do_count = ref false +let do_expand = ref false +let range = ref "" + +let _ = + let warn_hash = Hashtbl.create 1733 in + let add_warning warn_type node = + if Hashtbl.mem warn_hash warn_type then + let new_list = node :: (Hashtbl.find warn_hash warn_type) in + Hashtbl.replace warn_hash warn_type new_list + else + Hashtbl.add warn_hash warn_type (node::[]) in + + let print_warnings () = + let node_list nl = + let ary_nodes = Array.of_list nl in + compress_range ary_nodes in + + let print_warn w nl = + print_endline (" " ^ w ^ ": " ^ (node_list nl)) in + + if Hashtbl.length warn_hash > 0 then + begin + print_endline "WARNINGS"; + Hashtbl.iter + (fun k v -> print_warn k v) + warn_hash + end in + + let args = [ ("--count", Arg.Set do_count, "\tcount the nodes"); + ("-c", Arg.Set do_count, "\t\tcount the nodes"); + ("--expand", Arg.Set do_expand, "\tprint the nodes one per line"); + ("-e", Arg.Set do_expand, "\t\tprint the nodes one per line") + ] in + Arg.parse args (fun r -> range := r) "Usage: range [options] \nWhere options are:"; + want_caching true; + want_warnings true; + set_warning_callback add_warning; + + if !range = "" then print_endline "Need a range to parse." + else begin + let nodes = expand_range !range in + if !do_count then + Printf.printf "%d\n" (Array.length nodes) + else if !do_expand then + Array.iter print_endline nodes + else + print_endline (compress_range nodes); + end; + + print_warnings ();; + + diff --git a/librange/source/netblocks.ml b/librange/source/netblocks.ml new file mode 100644 index 0000000..97f5e52 --- /dev/null +++ b/librange/source/netblocks.ml @@ -0,0 +1,37 @@ +open Range_utils +open Memoize + +exception NetmaskErr of string +let netmask_colo_memo = + memoize ( + fun () -> + let netmask_colo = Hashtbl.create 611 in + let colo_netmask = Hashtbl.create 37 in + let lines = good_lines (read_file "/etc/yst-ip-list") in + + let rec parse_netblocks lines = + match lines with + | line :: rest -> + begin + let fields = Pcre.split line in + match fields with + | colo :: netmask_str :: rest -> + let netmask = Netmask.netmask_of_string netmask_str in + Hashtbl.add netmask_colo netmask (netmask_str, colo); + Hashtbl.add colo_netmask colo netmask_str + | _ -> raise (NetmaskErr ("yst-ip-list: " ^ line)) + end; + parse_netblocks rest + | [] -> () in + parse_netblocks lines; + netmask_colo, colo_netmask) + +let find_netblock ip = + let netmask_colo, _ = netmask_colo_memo.get () in + Netmask.find_netblock ip netmask_colo + +let vlans_for_dc dc = + let _, colo_netmask = netmask_colo_memo.get () in + Hashtbl.find_all colo_netmask dc + +let clear_cache () = netmask_colo_memo.clear () diff --git a/librange/source/netblocks.mli b/librange/source/netblocks.mli new file mode 100644 index 0000000..e521754 --- /dev/null +++ b/librange/source/netblocks.mli @@ -0,0 +1,3 @@ +val find_netblock : Tinydns.ip -> (string * string) option +val clear_cache : unit -> unit +val vlans_for_dc : string -> string list diff --git a/librange/source/netmask.ml b/librange/source/netmask.ml new file mode 100644 index 0000000..93e6485 --- /dev/null +++ b/librange/source/netmask.ml @@ -0,0 +1,59 @@ +open Scanf +open Int64 +exception NetmaskErr of string + +let imask_array = Array.make 33 0L +let init_imask = + let imask n = + sub + (shift_left 1L 32) + (shift_left 1L (32 - n)) in + for i = 0 to 32 do + imask_array.(i) <- imask i + done + +type netmask = { base: int64; bits: int } +(* return a binary representation of the netmask *) +let netmask_of_string netmask = + let netmask_parts = + let rex = Pcre.regexp "^(\\d+\\.\\d+\\.\\d+\\.\\d+)/(\\d+)$" in + try + Pcre.extract ~rex ~full_match:false netmask + with Not_found -> + let rex = Pcre.regexp "^\\d+\\.\\d+\\.\\d+\\.\\d+$" in + if Pcre.pmatch ~rex netmask then + [| netmask ; "32" |] + else + raise (NetmaskErr ("netmask: " ^ netmask)) in + let net_base = Tinydns.quad2int netmask_parts.(0) and + net_bits = int_of_string netmask_parts.(1) in + let net_mask = imask_array.(net_bits) in + { base=logand net_base net_mask; bits = net_bits } + +let string_of_netmask netmask = + let b = netmask.base in + Printf.sprintf "%d.%d.%d.%d/%d" + (to_int (shift_right (logand b 0xff000000L) 24)) + (to_int (shift_right (logand b 0x00ff0000L) 16)) + (to_int (shift_right (logand b 0x0000ff00L) 8)) + (to_int (logand b 0x000000ffL)) + netmask.bits + +let host_in_netblock netblock ip = + let binary_ip = Tinydns.int_of_ip ip and + imask = imask_array.(netblock.bits) in + (logand binary_ip imask) = netblock.base + +(* find the netblock for the given IP *) +let find_netblock ip hash = + let binary_ip = Tinydns.int_of_ip ip in + let rec find_it n bits = + if bits < 0 then None + else + let search_bits = logand n (imask_array.(bits)) in + let net = { base=search_bits; bits=bits } in + if Hashtbl.mem hash net then + Some (Hashtbl.find hash net) + else + find_it n (bits - 1) in + find_it binary_ip 32 diff --git a/librange/source/netmask.mli b/librange/source/netmask.mli new file mode 100644 index 0000000..3eb526c --- /dev/null +++ b/librange/source/netmask.mli @@ -0,0 +1,5 @@ +type netmask +val netmask_of_string : string -> netmask +val string_of_netmask : netmask -> string +val find_netblock : Tinydns.ip -> (netmask, 'a) Hashtbl.t -> 'a option +val host_in_netblock : netmask -> Tinydns.ip -> bool diff --git a/librange/source/parser.mly b/librange/source/parser.mly new file mode 100644 index 0000000..770eaa2 --- /dev/null +++ b/librange/source/parser.mly @@ -0,0 +1,52 @@ +%token RANGE +%token LITERAL +%token FUNCTION +%token REGEX FILTER +%token HOSTGROUP +%token UNION DIFF INTER INTER_HP NOT +%token ADMIN GET_CLUSTER GET_GROUP FUNCTION +%token CLUSTER_EXP COLON +%token LPAREN RPAREN LBRACE RBRACE +%token EOL +%left FUNCTION +%left UNION DIFF INTER /* lowest precendence */ +%left INTER_HP /* inter. medium precedence */ +%left NOT /* negation operator */ +%nonassoc CLUSTER_EXP HOSTGROUP ADMIN GET_CLUSTER GET_GROUP /* highest */ +%left COLON +%left LBRACE RBRACE + +%start main +%type main /* the parser will return an AST */ +%% + +main: + rangeexpr EOL { $1 } +; + +rangeexpr: + | { Ast.EmptyExpr } + | LITERAL { Ast.Literal $1 } + | LITERAL LITERAL { Ast.Literal ($1 ^ $2) } + | RANGE { match $1 with (prefix, start, finish, domain) -> + Ast.Range (prefix, start, finish, domain) } + | LPAREN rangeexpr RPAREN { Ast.Parens $2 } + | rangeexpr UNION rangeexpr { Ast.Union ($1, $3) } + | rangeexpr LBRACE rangeexpr RBRACE rangeexpr { Ast.Braces ($1, $3, $5) } + | REGEX { Ast.Regex $1 } + | rangeexpr DIFF rangeexpr { Ast.Diff ($1, $3) } + | rangeexpr INTER rangeexpr { Ast.Inter ($1, $3) } + | rangeexpr INTER_HP rangeexpr { Ast.Inter ($1, $3) } + | CLUSTER_EXP rangeexpr COLON rangeexpr + { Ast.Cluster ($2, $4) } + | CLUSTER_EXP rangeexpr { Ast.Cluster ($2, (Ast.Literal "CLUSTER")) } + | ADMIN rangeexpr { Ast.Admin $2 } + | HOSTGROUP rangeexpr { Ast.HostGroup $2 } + | GET_CLUSTER rangeexpr { Ast.GetCluster $2 } + | NOT rangeexpr { + Ast.Diff ((Ast.Cluster ((Ast.Literal "GROUPS"), (Ast.Literal "ALL"))), + $2) } + | GET_GROUP rangeexpr { Ast.GetGroup $2 } + | FUNCTION { Ast.Function (fst($1), snd($1)) } +; + diff --git a/librange/source/range.h b/librange/source/range.h new file mode 100644 index 0000000..65439cc --- /dev/null +++ b/librange/source/range.h @@ -0,0 +1,18 @@ + +#define LIBRANGE_VERSION "2.0.0" + +void range_startup(void); +const char ** range_expand(const char * c_range); +const char ** range_expand_sorted(const char * c_range); +char * range_compress(const char ** c_nodes, const char * c_separator); +char * range_parse(const char * c_range); +void range_set_altpath(const char * c_path); +void range_free_nodes(const char ** p); +void range_clear_caches(); + +void range_want_caching(int); +void range_want_warnings(int); + +char * range_get_exception(); +void range_clear_exception(); +char * range_get_version(); diff --git a/librange/source/range.ml b/librange/source/range.ml new file mode 100644 index 0000000..c9a6b68 --- /dev/null +++ b/librange/source/range.ml @@ -0,0 +1,13 @@ +open Evaluate + +let parse_range range = compress_range (expand_range range);; + +Callback.register "ocaml_parse_range" parse_range;; +Callback.register "ocaml_expand_range" expand_range;; +Callback.register "ocaml_compress_range" compress_range;; +Callback.register "ocaml_sorted_expand_range" sorted_expand_range;; +Callback.register "ocaml_range_set_altpath" range_set_altpath;; +Callback.register "ocaml_want_caching" want_caching;; +Callback.register "ocaml_want_warnings" want_warnings;; +Callback.register "ocaml_clear_caches" clear_caches;; + diff --git a/librange/source/range_utils.ml b/librange/source/range_utils.ml new file mode 100644 index 0000000..125953f --- /dev/null +++ b/librange/source/range_utils.ml @@ -0,0 +1,118 @@ +let do_warnings = ref false + +let want_warnings t = + let prev_value = !do_warnings in + do_warnings := t; + prev_value + +(* print a warning if the user called 'want_warnings true' *) +let print_warning warn_type node = + Printf.eprintf "%s: %s\n" warn_type node; + flush stderr + +let warning_callback = ref print_warning + +let set_warning_callback f = warning_callback := f + +let do_warning warn_type node = + if !do_warnings then + !warning_callback warn_type node + +let is_dir x = + let st = Unix.stat x in + st.Unix.st_kind = Unix.S_DIR;; + +let is_file x = + try + let st = Unix.stat x in + st.Unix.st_kind = Unix.S_REG + with Unix.Unix_error _ -> false + +(* like sprintf %0d n *) +let pad0 n len = + let s = string_of_int n in + let ls = String.length s in + let b = Buffer.create 16 in + for i = ls to (len - 1) do + Buffer.add_char b '0' + done; + Buffer.add_string b s; + Buffer.contents b + +(* get the keys of a hash as a list *) +let hash_keys hash = Hashtbl.fold (fun k _ acc -> k::acc) hash [] + +(* true if 'str' begins with 'substr' *) +let begins_with substr str = + let l1 = String.length substr and + l2 = String.length str in + if l1 > l2 then false + else + let leftstr = String.sub str 0 l1 in + leftstr = substr + +(* true if 'str' ends with 'substr' *) +let ends_with substr str = + let l1 = String.length substr and + l2 = String.length str in + if l1 > l2 then false + else + let rightstr = String.sub str (l2 - l1) l1 in + rightstr = substr + +(* read a file returning a list of lines *) +let read_file filename = + let ch = open_in filename and + lines = ref [] in + let rec loop () = + let ln = input_line ch in + lines := ln :: !lines; + loop () in + try + loop (); + with + End_of_file -> close_in ch; List.rev !lines + +(* read a file - following $INCLUDE if needed *) +let rec read_big_file filename = + let get_reldir_name fn = + try (String.sub fn 0 (String.rindex fn '/')) ^ "/" with _ -> "" and + get_file_name ln = + let inc_re = Pcre.regexp "\\$INCLUDE\\s+\"([^\"]+)\"" in + (Pcre.extract ~full_match:false ~rex:inc_re ln).(0) and + ch = open_in filename and + lines = ref [] in + try + let rec loop () = + let ln = input_line ch in + if begins_with "$INCLUDE" ln then + begin + let inc_fn = get_file_name ln in + let included_file_name = + if inc_fn.[0] = '/' then inc_fn + else (get_reldir_name filename) ^ inc_fn in + lines := (List.rev + (read_big_file included_file_name)) @ + !lines + end + else + lines := ln::!lines; + loop () in + loop () + with + End_of_file -> close_in ch; List.rev !lines + +(* return a list of lines that are not comments and not blank *) +let good_lines lines = + let comment_re = Pcre.regexp "#.*" and + end_blank_re = Pcre.regexp "\\s+$" in + let fix_line line = Pcre.replace ~rex:end_blank_re + (Pcre.replace ~rex:comment_re line) in + List.filter (fun l -> String.length l > 0) + (List.map fix_line lines) + +let hash_keys h = + Hashtbl.fold (fun k _ acc -> k :: acc) h [] + +let hash_values h = + Hashtbl.fold (fun _ v acc -> v :: acc) h [] diff --git a/librange/source/range_utils.mli b/librange/source/range_utils.mli new file mode 100644 index 0000000..edadaf8 --- /dev/null +++ b/librange/source/range_utils.mli @@ -0,0 +1,38 @@ +(* show warnings true/false *) +val want_warnings : bool -> bool + +(* print a warning if the user said 'want_warnings true' *) +val do_warning : string -> string -> unit + +(* instead of printing the warning, just call this function instead *) +val set_warning_callback : (string -> string -> unit) -> unit + +(* -d *) +val is_dir : string -> bool + +(* -e *) +val is_file : string -> bool + +(* like sprintf %0d n *) +val pad0 : int -> int -> string + +(* get the keys of a hash a list *) +val hash_keys : ('a, 'b) Hashtbl.t -> 'a list + +(* begins_with *) +val begins_with : string -> string -> bool +(* ends_with *) +val ends_with : string -> string -> bool + +(* read a file, returning a list of lines *) +val read_file : string -> string list + +(* read a file following $INCLUDE directives *) +val read_big_file : string -> string list + +(* return a list of lines that are not comments and not blank *) +val good_lines : string list -> string list + +val hash_keys : ('a, 'b) Hashtbl.t -> 'a list +val hash_values : ('a, 'b) Hashtbl.t -> 'b list + diff --git a/librange/source/rip_objects_hack.sh b/librange/source/rip_objects_hack.sh new file mode 100644 index 0000000..7bcb607 --- /dev/null +++ b/librange/source/rip_objects_hack.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +mkdir a b +cd a +ar x $1/libunix.a +cd .. +cd b +ar x $1/libasmrun.a diff --git a/librange/source/testrange-reps.c b/librange/source/testrange-reps.c new file mode 100644 index 0000000..a8dee29 --- /dev/null +++ b/librange/source/testrange-reps.c @@ -0,0 +1,23 @@ +#include +#include +#include +#include + +#include "range.h" + + +int main(int argc, char **argv) { + struct nodes_t *result; + long int i; + char * compressed; + if (argc < 2) { + printf("argc is %d, needs args\n", argc); + exit(1); + } + range_startup(); + printf("hello world\n"); + while (1) { + result = range_expand(argv[1]); + } + return 0; +} diff --git a/librange/source/testrange.c b/librange/source/testrange.c new file mode 100644 index 0000000..c35e2a7 --- /dev/null +++ b/librange/source/testrange.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include + +#include "range.h" + + +int main(int argc, char **argv) { + struct nodes_t *result; + long int i; + char * compressed; + if (argc < 2) { + printf("argc is %d, needs args\n", argc); + exit(1); + } + range_startup(); + printf("hello world\n"); + result = expand_range(argv[1]); + printf("done with expand range\n"); + if (result) { + printf("got %d nodes\n", result->length); + for (i = 0; i < result->length ; i++) { + printf("string: '%s'\n", result->nodes[i]); + } + } else { + printf("got exception!\n"); + fflush(NULL); + printf("got ex, index: %d\n", get_exception()); + fflush(NULL); + printf("got threw ex '%s'\n", get_exception_string()); + exit(88); + } + printf("now to compress it:\n"); + compressed = compress_range(result); + free_nodes(result); + printf("freed result ok!\n"); + if (compressed) { + printf("compressed: '%s'\n", compressed); + } else { + printf("compressed threw exception\n"); + exit(89); + } + free(compressed); + printf("freed compressed string\n"); + result = sorted_expand_range(argv[1]); + for (i = 0; i < result->length ; i++) { + printf("sort: '%s'\n", result->nodes[i]); + } + free_nodes(result); + printf("freed result ok!\n"); + set_range_altpath("/tmp"); + /* printf("parsed %%ks301,&/0$/: '%s'\n", parse_range("%ks301,&/0$/")); */ + printf("sleeping for 5 seconds ... "); + fflush(NULL); + sleep(5); + printf("now setting up for big loop\n"); + for (i = 0; i < 30000; i++) { + result = expand_range(argv[1]); + free_nodes(result); + } + printf("sleeping for 10 more seconds ... "); + fflush(NULL); + sleep(10); + printf("done!\n"); + return 0; +} diff --git a/librange/source/tinydns.ml b/librange/source/tinydns.ml new file mode 100644 index 0000000..fd11bf9 --- /dev/null +++ b/librange/source/tinydns.ml @@ -0,0 +1,106 @@ +(* parse the tinydns root/data file *) +open Memoize +open Range_utils +open Int64 + +type ip = { binary: int64; repr: string } + +(* have to do 64 bit arithmetic because 32 bit ints are signed + and the default int uses an extra bit so we only have 30 usable bits *) +let quad2int ip = + Scanf.sscanf ip "%d.%d.%d.%d" + (fun a b c d -> + (add + (add + (shift_left (of_int a) 24) + (shift_left (of_int b) 16)) + (add + (shift_left (of_int c) 8) + (of_int d)))) + +let ip_of_string str = + { binary=quad2int str; repr=str } + +let int_of_ip ip = ip.binary +let string_of_ip ip = ip.repr + +let dns_read_memo = + memoize ( + fun () -> + let hosts_ip = Hashtbl.create 99997 in + let cnames = Hashtbl.create 23433 in + let ch = open_in "/etc/service/tinydns/root/data" in + let a_rec_re = Pcre.regexp "^\\+([^:]+):([\\d.]+):0" in + let cname_rec_re = Pcre.regexp "^C([^:]+):([^:]+)\\.:0" in + let rec loop () = + let ln = input_line ch in + begin + try + let res = Pcre.extract ~full_match:false ~rex:a_rec_re ln in + Hashtbl.add hosts_ip res.(0) (ip_of_string res.(1)) + with Not_found -> ( + try + let res = Pcre.extract ~full_match:false ~rex:cname_rec_re ln in + Hashtbl.add cnames res.(0) res.(1) + with Not_found -> (); + ); + end; + loop() in + try + loop (); + with + End_of_file -> close_in ch; + (hosts_ip, cnames));; + +let fqdn host = + if ends_with ".com" host then host + else if ends_with ".net" host then host + else host ^ ".inktomisearch.com" + +(* If the first character is a digit, we return it unmodified, since + chances are it is an IP. + Maybe I should do a full blown regex (^[\d.]+$) - and actually benchmark + the cost of it but for now this will do *) +let get_ip host = + if host.[0] >= '0' && host.[0] <= '9' then Some (ip_of_string host) + else + let (a, c) = dns_read_memo.get () in + let rec resolve_cname host = + if Hashtbl.mem c host then + resolve_cname (Hashtbl.find c host) + else + host in + let real_name = resolve_cname (fqdn host) in + try + Some (Hashtbl.find a real_name) + with Not_found -> + do_warning "NOTINYDNS" real_name; + None + + +let clean_up hostname = + let remove_domain host domain = + let n = String.length host in + if ends_with domain host then + String.sub host 0 (n - (String.length domain)) + else + host in + let n = String.length hostname in + let final_char = hostname.[n - 1] in + let host_no_dot = + if final_char = '.' then + String.sub hostname 0 (n - 1) + else + hostname in + let inkt = remove_domain host_no_dot ".inktomisearch.com" in + let yst = remove_domain inkt ".yst.corp.yahoo.com" in + yst + +(* return a list of all the IPs (with the hostnames associated with them) *) +let all_ip_hosts () = + match dns_read_memo.get() with + a_records, _ -> + Hashtbl.fold (fun k v a -> (v,clean_up k) :: a) a_records [] + +let clear_cache () = dns_read_memo.clear () + diff --git a/librange/source/tinydns.mli b/librange/source/tinydns.mli new file mode 100644 index 0000000..a3a1e4f --- /dev/null +++ b/librange/source/tinydns.mli @@ -0,0 +1,8 @@ +type ip +val get_ip : string -> ip option +val clear_cache : unit -> unit +val fqdn : string -> string +val all_ip_hosts : unit -> (ip * string) list +val int_of_ip : ip -> int64 +val string_of_ip : ip -> string +val quad2int : string -> int64 diff --git a/mod_range/index.yaml b/mod_range/index.yaml new file mode 100644 index 0000000..a08bee4 --- /dev/null +++ b/mod_range/index.yaml @@ -0,0 +1,9 @@ +--- +default: + name: mod_range + requires: + - libcrange + - httpd + - lwes-local + summary: mod_range - apache module for range + version: '1.1.0' diff --git a/mod_range/root/etc/httpd/conf.d/mod_range.conf b/mod_range/root/etc/httpd/conf.d/mod_range.conf new file mode 100644 index 0000000..ef0187f --- /dev/null +++ b/mod_range/root/etc/httpd/conf.d/mod_range.conf @@ -0,0 +1,15 @@ +LoadModule range_module modules/mod_range.so + + + SetHandler server-range + + +RangeLogRequests On +RangeTimeToLive 3600 +RangeRequestsToServe 500 + +RangeLogLwes On +RangeLwesAddr "239.11.50.2" +RangeLwesPort 2011 +RangeLwesTimeToLive 16 +RangeLwesHostHeader "NS-Client-IP" diff --git a/mod_range/scripts/build b/mod_range/scripts/build new file mode 100755 index 0000000..3a0604e --- /dev/null +++ b/mod_range/scripts/build @@ -0,0 +1,17 @@ +#!/bin/sh + +PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin + +apxs -c mod_range.c -lcrange + +platform=`uname -i` +if [ $platform = i386 ]; then + dir=/usr/lib/httpd/modules +else + dir=/usr/lib64/httpd/modules +fi + +mkdir -p $DESTDIR/$dir +install -o root -g root -m 0755 .libs/mod_range.so $DESTDIR/$dir + + diff --git a/mod_range/scripts/post.sh b/mod_range/scripts/post.sh new file mode 100755 index 0000000..5fe7816 --- /dev/null +++ b/mod_range/scripts/post.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/etc/init.d/httpd restart diff --git a/mod_range/source/Makefile.regular b/mod_range/source/Makefile.regular new file mode 100644 index 0000000..d0b4d11 --- /dev/null +++ b/mod_range/source/Makefile.regular @@ -0,0 +1,6 @@ +all: + apxs -c -I/home/y/include/lwes-0 mod_range.c -lcrange -L/home/y/lib64 -llwes + +install: + apxs -c -i -I/home/y/include/lwes-0 mod_range.c -lcrange -L/home/y/lib64 -llwes + /etc/init.d/httpd restart diff --git a/mod_range/source/mod_yahoo_range.c b/mod_range/source/mod_yahoo_range.c new file mode 100644 index 0000000..4bfd240 --- /dev/null +++ b/mod_range/source/mod_yahoo_range.c @@ -0,0 +1,385 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms +*/ + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +static struct lwes_emitter *emitter; +static int log_requests = 0; +static int range_ttl = 3600; +static int range_rtl = 2000; +static int log_lwes = 0; +static int range_lwes_ttl = 16; +static char *range_lwes_addr = NULL; +static int range_lwes_port = 0; +static int range_lwes_error = 0; +static char *range_lwes_host_header = NULL; + +/* time in seconds when this child started */ +static time_t time_started = 0; + +/* Emit an LWES event describing this request */ +static void send_lwes(request_rec * r, struct timeval *diff, int err, int warn) +{ + struct lwes_event *event; + const char *userAgent = "UNKNOWN"; + const char *client_ip = NULL; + + /* If we previously failed to create the LWES emitter, don't keep trying */ + if (range_lwes_error) + return; + + /* We create the LWES emitter the first time through. */ + if (emitter == NULL) { + emitter = lwes_emitter_create_with_ttl(range_lwes_addr, "0.0.0.0", + range_lwes_port, 0, 60, + range_lwes_ttl); + /* If we failed to create the emitter, log an error and mark a flag + so we don't continue to retry. */ + if (emitter == NULL) { + range_lwes_error = 1; + ap_log_rerror(APLOG_MARK, APLOG_CRIT, 0, r, + "Failed to create LWES emitter"); + return; + } + } + + /* Create an empty event for emission. */ + event = lwes_event_create(NULL, "Ranged::Serve"); + if (event == NULL) { + ap_log_rerror(APLOG_MARK, APLOG_CRIT, 0, r, + "Failed to create LWES event"); + return; + } + + /* Check the headers for some fields. */ + if (r->headers_in != NULL) { + /* Grab the user agent from the headers. If it wasn't passed in, + it'll default to "UNKNOWN". */ + userAgent = apr_table_get(r->headers_in, "User-Agent"); + if (userAgent == NULL) { + userAgent = "UNKNOWN"; + } + /* If we were asked to override the client IP with an HTTP header, + check for the header and fetch the value. */ + if (range_lwes_host_header != NULL) { + client_ip = apr_table_get(r->headers_in, range_lwes_host_header); + } + } + + /* If we were supposed to override the client IP with an HTTP header, + but the header didn't exist, OR if we aren't overriding, set the + client IP to the real remote connection IP. */ + if ((range_lwes_host_header != NULL && client_ip == NULL) || + range_lwes_host_header == NULL) { + client_ip = r->connection->remote_ip; + } + + /* Set the various LWES event fields that we'll be emitting. */ + lwes_event_set_STRING(event, "ua", userAgent); + lwes_event_set_STRING(event, "client", client_ip); + lwes_event_set_U_INT_16(event, "serve", 1); + lwes_event_set_U_INT_16(event, "error", err); + lwes_event_set_U_INT_16(event, "warning", warn); + lwes_event_set_U_INT_64(event, "tts", + (diff->tv_sec * 1000000) + diff->tv_usec); + + /* Send the LWES event, and clean up. */ + lwes_emitter_emit(emitter, event); + lwes_event_destroy(event); +} + +static int timeval_subtract(struct timeval *result, struct timeval *end, + struct timeval *start) +{ + if (end->tv_usec < start->tv_usec) { + int nsec = (start->tv_usec - end->tv_usec) / 1000000 + 1; + start->tv_usec -= 1000000 * nsec; + start->tv_sec += nsec; + } + if (end->tv_usec - start->tv_usec > 1000000) { + int nsec = (end->tv_usec - start->tv_usec) / 1000000; + start->tv_usec += 1000000 * nsec; + start->tv_sec -= nsec; + } + result->tv_sec = end->tv_sec - start->tv_sec; + result->tv_usec = end->tv_usec - start->tv_usec; + return end->tv_sec < start->tv_sec; +} + +static char *read_post_data(request_rec * r) +{ + char* range = NULL; + char* range_ret = NULL; /* actual range which will be returned */ + int bytes_inserted; + apr_size_t bufsize; + apr_size_t post_data_size = 0; + + /* setup client to allow Apache to read request body, CHUNKED is supported :)*/ + if (ap_setup_client_block(r,REQUEST_CHUNKED_DECHUNK) != OK) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mod_range: ap_setup_client_block failed."); + return ""; + } + + /* Allocate 1MB initiially*/ + bufsize = 1024 * 1024; + range = (char*) malloc(bufsize+1); + + /*If client has data to send*/ + if( ap_should_client_block(r) ) { + while(1) { + /* read the data */ + bytes_inserted = ap_get_client_block(r, range, bufsize); + + if( bytes_inserted == 0 ) + break; + + if (bytes_inserted == -1) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mod_range: ap_get_client_block failed."); + free(range); + return ""; + } + + post_data_size += bytes_inserted; + + /*Allocate more if required, on > 7K*/ + if (post_data_size >= bufsize - (7 * 1024) ){ + bufsize += bufsize; + range = (char *) realloc(range,bufsize); + } + } /* end of while(1) */ + range[post_data_size] = '\0'; + } + /*copy range to range_ret*/ + range_ret = apr_pstrdup(r->pool,range); + /*unescape post params*/ + ap_unescape_url(range_ret); + free(range); + return range_ret; +} + +static int range_handler(request_rec * r) +{ + range_request *rr; + char *range; + int wants_list = 0; + int wants_expand = 0; + int warn = 0; + struct timeval t; + struct timeval end_t; + struct timeval diff_t; + timerclear(&end_t); + timerclear(&diff_t); + + if (strcmp(r->handler, "server-range")) + return DECLINED; + + if (r->method_number != M_GET && r->method_number != M_POST) { + if (log_lwes) + send_lwes(r, &diff_t, 1, warn); + return HTTP_METHOD_NOT_ALLOWED; + } + + wants_list = strcmp(r->path_info, "/list") == 0; + if (!wants_list) + wants_expand = strcmp(r->path_info, "/expand") == 0; + + if (!wants_list && !wants_expand) + return DECLINED; + + if (log_requests || log_lwes || !time_started) { + gettimeofday(&t, NULL); + if (!time_started) + time_started = t.tv_sec; + } + + ap_set_content_type(r, "text/plain"); + if (r->method_number == M_GET) { + if(r->args != NULL ) { + /* unescape GET params*/ + ap_unescape_url(r->args); + range = r->args; + } + else + range = ""; + } + else + range = read_post_data(r); + + rr = range_expand(NULL, r->pool, range); + gettimeofday(&end_t, NULL); + timeval_subtract(&diff_t, &end_t, &t); + + if (log_requests) { + double diff; + diff = end_t.tv_sec * 1000000.0 + end_t.tv_usec - + (t.tv_sec * 1000000.0 + t.tv_usec); + + diff /= 1E6; + ap_log_rerror(APLOG_MARK, APLOG_NOTICE, 0, r, + "%s -- %0.3fs", range, diff); + } + + if (range_request_has_warnings(rr)) { + warn = 1; + const char *warnings = range_request_warnings(rr); + char *header = (char *)warnings; + if (strlen(warnings) > 2048) { + header = apr_palloc(r->pool, 2048); + memcpy(header, warnings, 2047); + header[2047] = '\0'; + } + + apr_table_t *headers = r->headers_out; + apr_table_add(headers, "RangeException", header); + } + + if (wants_list) { + const char **nodes = range_request_nodes(rr); + while (*nodes) { + ap_rputs(*nodes++, r); + ap_rputc('\n', r); + } + } + else { + const char *compressed = range_request_compressed(rr); + ap_rputs(compressed, r); + } + + /* + if (--range_rtl < 1 || (end_t.tv_sec - time_started) > range_ttl) { + pid_t pid = getpid(); ap_log_rerror(APLOG_MARK, APLOG_NOTICE, 0, r, + "range gracefully restarting %u", pid); } */ + + if (log_lwes) + send_lwes(r, &diff_t, 0, warn); + + return OK; +} + +static const char *range_log_requests(cmd_parms * cmd, void *dummy, int flag) +{ + log_requests = flag; + return NULL; +} + +static const char *range_log_lwes(cmd_parms * cmd, void *dummy, int flag) +{ + log_lwes = flag; + return NULL; +} + +static const char *range_set_lwes_port(cmd_parms * cmd, void *dummy, + const char *arg) +{ + int port = atoi(arg); + if (port < 1) { + return "RangeLwesPort must be > 0"; + } + + range_lwes_port = port; + return NULL; +} + +static const char *range_set_lwes_addr(cmd_parms * cmd, void *dummy, + const char *arg) +{ + range_lwes_addr = strdup(arg); + return NULL; +} + +static const char *range_set_lwes_ttl(cmd_parms * cmd, void *dummy, + const char *arg) +{ + int ttl = atoi(arg); + if (ttl < 0) { + return "RangeLwesTimeToLive must be >= 0"; + } + + range_lwes_ttl = ttl; + return NULL; +} + +static const char *range_set_lwes_host_header(cmd_parms * cmd, void *dummy, + const char *arg) +{ + range_lwes_host_header = strdup(arg); + return NULL; +} + +static const char *range_set_ttl(cmd_parms * cmd, void *dummy, const char *arg) +{ + int ttl = atoi(arg); + if (ttl < 1) { + return "RangeTimeToLive must be > 0"; + } + + range_ttl = ttl; + return NULL; +} + +static const char *range_set_rtl(cmd_parms * cmd, void *dummy, const char *arg) +{ + int rtl = atoi(arg); + if (rtl < 1) { + return "RangeRequestsToServe must be > 0"; + } + + range_rtl = rtl; + return NULL; +} + +static const command_rec config_range_cmds[] = { + AP_INIT_FLAG("RangeLogRequests", range_log_requests, NULL, RSRC_CONF, + "On or Off to enable or disable (default) logging"), + AP_INIT_FLAG("RangeLogLwes", range_log_lwes, NULL, RSRC_CONF, + "On or Off to enable or disable (default) " + "logging via LWES emission"), + AP_INIT_TAKE1("RangeLwesAddr", range_set_lwes_addr, NULL, RSRC_CONF, + "Multicast address for sending LWES events"), + AP_INIT_TAKE1("RangeLwesPort", range_set_lwes_port, NULL, RSRC_CONF, + "Multicast port for sending LWES events"), + AP_INIT_TAKE1("RangeLwesTimeToLive", range_set_lwes_ttl, NULL, RSRC_CONF, + "TTL setting for emitted multicast packets (default 16)"), + AP_INIT_TAKE1("RangeLwesHostHeader", range_set_lwes_host_header, NULL, + RSRC_CONF, "Override LWES-logged client IP using this header" + " (defaults to unused, if header does not exist, real client" + " IP will be used)"), + AP_INIT_TAKE1("RangeTimeToLive", range_set_ttl, NULL, RSRC_CONF, + "the number of seconds an httpd child should live " + "(default 3600)"), + AP_INIT_TAKE1("RangeRequestsToServe", range_set_rtl, NULL, RSRC_CONF, + "the number of requests an httpd child should serve " + "(default 2000)"), + {NULL} +}; + +static void register_hooks(apr_pool_t * p) +{ + ap_hook_handler(range_handler, NULL, NULL, APR_HOOK_MIDDLE); +} + +module AP_MODULE_DECLARE_DATA range_module = { + STANDARD20_MODULE_STUFF, + NULL, /* create per-directory config structure */ + NULL, /* merge per-directory config structures */ + NULL, /* create per-server config structure */ + NULL, /* merge per-server config structures */ + config_range_cmds, /* command apr_table_t */ + register_hooks /* register hooks */ +}; diff --git a/mod_range/source/mod_yahoo_range.conf b/mod_range/source/mod_yahoo_range.conf new file mode 100644 index 0000000..f783179 --- /dev/null +++ b/mod_range/source/mod_yahoo_range.conf @@ -0,0 +1,17 @@ +LoadModule range_module libexec/apache/apache_mod_yahoo_range.so + + + SetHandler server-range + + +RangeLogRequests On +RangeTimeToLive 3600 +RangeRequestsToServe 500 + +# CHANGE THESE +RangeLogLwes On +RangeLwesAddr "239.11.50.1" +RangeLwesPort 2011 +RangeLwesTimeToLive 16 +RangeLwesHostHeader "Custom Host Header for L7 SNAT vips Goes Here" + diff --git a/perl_seco_data_range/index.yaml b/perl_seco_data_range/index.yaml new file mode 100644 index 0000000..dab3029 --- /dev/null +++ b/perl_seco_data_range/index.yaml @@ -0,0 +1,14 @@ +default: + name: perl-seco-data-range + summary: "Perl interface to libcrange" + arch: noarch + version: '1.1' + requires: + - perl-seco-getopt +yinst: + name: perl_seco_data_range + summary: "Perl interface to libcrange" + arch: noarch + version: 0.0.4 + requires: + - perl_seco_getopt diff --git a/perl_seco_data_range/root/usr/bin/er b/perl_seco_data_range/root/usr/bin/er new file mode 100755 index 0000000..51fe64e --- /dev/null +++ b/perl_seco_data_range/root/usr/bin/er @@ -0,0 +1,69 @@ +#!/usr/local/bin/perl -w + +use strict; +use warnings 'all'; +use Seco::Data::Range; +use Seco::Getopt; + +my $opt = Seco::Getopt->new( + options => { + 'e|expand' => 'Print one node per line', + 'c|count' => 'Print the count, not the expansion', + 'v|vip=s' => 'Use this vip', + 'p|port=i' => 'Port to use when connecting', + 't|timeout=i' => 'Timeout value for queries', + }, + description => "\nUsage: $0 [options] \n" . + "If is '-' then it will read the range(s)" . + " from standard input", +); + + +my %opts; +$opts{server} = $opt->get('v') if ($opt->get('v')); +$opts{timeout} = $opt->get('t') if ($opt->get('t')); +$opts{port} = $opt->get('p') if ($opt->get('p')); +$opts{list} = 1; + +my $range = Seco::Data::Range->new ( %opts ); + +my $r = join(',', @ARGV); +my @nodes; +if ($r eq '') { + warn "WARN: No range given!\n"; + exit 1; +} +elsif ($r eq '-') { + my @slurp = ; + chomp for @slurp; + @nodes = $range->expand(join ',',@slurp); +} +else { + @nodes = $range->expand($r); +} + +# capture the error from the initial expansion +my $expansion_err = $range->last_err; + +if ($opt->get('c')) { + print scalar @nodes, "\n"; +} +else { + if (@nodes) { + if ($opt->get('e')) { + print join( "\n", @nodes ), "\n"; + } + else { + print $range->compress( \@nodes ), "\n"; + } + warn "WARN: " . $expansion_err . "\n" if $expansion_err; + } + else { + warn "No nodes specified.\n"; + warn "WARN: " . $expansion_err . "\n" if $expansion_err; + exit 1; + } + +} +exit 0; + diff --git a/perl_seco_data_range/root/usr/share/man/man1/er.1 b/perl_seco_data_range/root/usr/share/man/man1/er.1 new file mode 100644 index 0000000..042f071 --- /dev/null +++ b/perl_seco_data_range/root/usr/share/man/man1/er.1 @@ -0,0 +1,186 @@ +.\" Automatically generated by Pod::Man 2.17 (Pod::Simple 3.07) +.\" +.\" Standard preamble: +.\" ======================================================================== +.de Sh \" Subsection heading +.br +.if t .Sp +.ne 5 +.PP +\fB\&\\$1\fR +.PP +.. +.de Sp \" Vertical space (when we can't use .PP) +.if t .sp .5v +.if n .sp +.. +.de Vb \" Begin verbatim text +.ft CW +.nf +.ne \\$1 +.. +.de Ve \" End verbatim text +.ft R +.fi +.. +.\" Set up some character translations and predefined strings. \*(-- will +.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left +.\" double quote, and \*(R" will give a right double quote. \*(C+ will +.\" give a nicer C++. Capital omega is used to do unbreakable dashes and +.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, +.\" nothing in troff, for use with C<>. +.tr \(*W- +.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' +.ie n \{\ +. ds -- \(*W- +. ds PI pi +. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch +. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch +. ds L" "" +. ds R" "" +. ds C` "" +. ds C' "" +'br\} +.el\{\ +. ds -- \|\(em\| +. ds PI \(*p +. ds L" `` +. ds R" '' +'br\} +.\" +.\" Escape single quotes in literal strings from groff's Unicode transform. +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.\" +.\" If the F register is turned on, we'll generate index entries on stderr for +.\" titles (.TH), headers (.SH), subsections (.Sh), items (.Ip), and index +.\" entries marked with X<> in POD. Of course, you'll have to process the +.\" output yourself in some meaningful fashion. +.ie \nF \{\ +. de IX +. tm Index:\\$1\t\\n%\t"\\$2" +.. +. nr % 0 +. rr F +.\} +.el \{\ +. de IX +.. +.\} +.\" +.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). +.\" Fear. Run. Save yourself. No user-serviceable parts. +. \" fudge factors for nroff and troff +.if n \{\ +. ds #H 0 +. ds #V .8m +. ds #F .3m +. ds #[ \f1 +. ds #] \fP +.\} +.if t \{\ +. ds #H ((1u-(\\\\n(.fu%2u))*.13m) +. ds #V .6m +. ds #F 0 +. ds #[ \& +. ds #] \& +.\} +. \" simple accents for nroff and troff +.if n \{\ +. ds ' \& +. ds ` \& +. ds ^ \& +. ds , \& +. ds ~ ~ +. ds / +.\} +.if t \{\ +. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" +. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' +. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' +. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' +. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' +. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' +.\} +. \" troff and (daisy-wheel) nroff accents +.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' +.ds 8 \h'\*(#H'\(*b\h'-\*(#H' +.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] +.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' +.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' +.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] +.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] +.ds ae a\h'-(\w'a'u*4/10)'e +.ds Ae A\h'-(\w'A'u*4/10)'E +. \" corrections for vroff +.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' +.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' +. \" for low resolution devices (crt and lpr) +.if \n(.H>23 .if \n(.V>19 \ +\{\ +. ds : e +. ds 8 ss +. ds o a +. ds d- d\h'-1'\(ga +. ds D- D\h'-1'\(hy +. ds th \o'bp' +. ds Th \o'LP' +. ds ae ae +. ds Ae AE +.\} +.rm #[ #] #H #V #F C +.\" ======================================================================== +.\" +.IX Title "er 1" +.TH er 1 "2008-12-22" "Yahoo" "Yahoo internal" +.\" For nroff, turn off justification. Always turn off hyphenation; it makes +.\" way too many mistakes in technical documents. +.if n .ad l +.nh +.SH "NAME" +er +.SH "SYNOPSIS" +.IX Header "SYNOPSIS" +.IP "er [options] " 4 +.IX Item "er [options] " +.PD 0 +.IP "er \-\-help" 4 +.IX Item "er --help" +.PD +.SH "DESCRIPTION" +.IX Header "DESCRIPTION" +er displays the expansion of . Any valid range expression can be used. +.PP +If is '\-' then it will read the range(s) from standard input. +.PP +A handy trick, you could have a list of hosts, pipe that through \*(L"er \-\*(R", and er would output the brace-expanded range expression that the list comprises. +.SH "OPTIONS" +.IX Header "OPTIONS" +.IP "\-c, \-\-count" 4 +.IX Item "-c, --count" +Print the count, not the expansion, this is a shortcut for doing something like \f(CW\*(C`er \-e \*(Aqflatten(foo)\*(Aq | wc \-l\*(C'\fR +.IP "\-e, \-\-expand" 4 +.IX Item "-e, --expand" +The default output is to group hosts in glob-like patterns, i.e. {ks121\-9,ks1210\-9}.ysm.ac4.yahoo.com. +This option overrides that behavior, and results in a list of all hostnames being displayed, one per line. +.IP "\-p, \-\-port" 4 +.IX Item "-p, --port" +Port of the range server to use when connecting. +.IP "\-t, \-\-timeout" 4 +.IX Item "-t, --timeout" +Timeout value for queries to the range server. +.IP "\-v, \-\-vip" 4 +.IX Item "-v, --vip" +Use this vip, instead of the default range server \*(L"range\*(R" +.SH "EXAMPLES" +.IX Header "EXAMPLES" +.Vb 2 +\& /usr/local/bin/er \-e $range +\& echo "there are $(/usr/local/bin/er \-c $range) names in $range" +.Ve +.SH "BUGS" +.IX Header "BUGS" +None noted. +.SH "SEE ALSO" +.IX Header "SEE ALSO" +\&\fIrange\fR\|(7), Seco::Range diff --git a/perl_seco_data_range/scripts/build b/perl_seco_data_range/scripts/build new file mode 100755 index 0000000..934bbaf --- /dev/null +++ b/perl_seco_data_range/scripts/build @@ -0,0 +1,12 @@ +#!/bin/sh + +$PERL Makefile.PL && make && make test && make install DESTDIR=$DESTDIR || exit 1 + +find $DESTDIR -name perllocal.pod -type f -exec rm \{\} \; + +# create README file from POD doc +pod2text lib/Seco/Data/Range.pm >README +# XXX: work around missing perldispatch in /usr/local +if [ $? -ne 0 ]; then + pod2text-5.8 lib/Seco/Data/Range.pm >README || exit 1 +fi diff --git a/perl_seco_data_range/source/Makefile.PL b/perl_seco_data_range/source/Makefile.PL new file mode 100644 index 0000000..58865fb --- /dev/null +++ b/perl_seco_data_range/source/Makefile.PL @@ -0,0 +1,13 @@ +use 5.006000; +use ExtUtils::MakeMaker; + +# install into version-independent path +use Config; +my $installsitelib = $Config{installsitelib}; +$installsitelib =~ s/\/[0-9\.]+$//; +die "version-independent libdir not found in \@INC" + unless (grep { $_ eq $installsitelib } (@INC)); + +WriteMakefile(NAME => 'Seco::Data::Range', + VERSION => '1.0', + INSTALLSITELIB => $installsitelib); diff --git a/perl_seco_data_range/source/README b/perl_seco_data_range/source/README new file mode 100644 index 0000000..bba2021 --- /dev/null +++ b/perl_seco_data_range/source/README @@ -0,0 +1,29 @@ +Seco-Data-Range version 1.00 +============================ + +OO Interface to a ranged instance. The default ranged server is +'boothost' unless the machine has a /service/ranged entry, in which +case 'localhost' is used. + +INSTALLATION + +To install this module type the following: + + perl Makefile.PL + make + make test + make install + +DEPENDENCIES + +This module requires perl v5.6 or later. + +AUTHOR + +Daniel Muino + +COPYRIGHT AND LICENCE + +Copyright (C) 2006 Yahoo! Inc. + + diff --git a/perl_seco_data_range/source/lib/Seco/Data/Range.pm b/perl_seco_data_range/source/lib/Seco/Data/Range.pm new file mode 100644 index 0000000..1d10c28 --- /dev/null +++ b/perl_seco_data_range/source/lib/Seco/Data/Range.pm @@ -0,0 +1,378 @@ +package Seco::Data::Range; + +# Copyright (c) 2011, Yahoo! Inc. All rights reserved. +# Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + + +use strict; +use warnings 'all'; + +use fields qw/server port timeout useragent last_err _ua list/; +use Carp; +use LWP::UserAgent; +use URI::Escape; + +our $VERSION = v1.0; + +my $node_regex = qr/ + ([-\w.]*?) # prefix + (\d+) # followed by the start of the range + (\.[-A-Za-z\d.]*[-A-Za-z]+[-A-Za-z\d.]*)? # optional domain +/x; + +sub new { + my $class = shift; + my __PACKAGE__ $self = fields::new($class); + my $default_server = 'range.ysm'; + if (@_ == 1) { + $default_server = shift; + } + + # override default arguments with user supplied hash + my %args = ( + 'server' => $default_server, + 'port' => 9999, + 'timeout' => 10, + 'useragent' => $0, + 'list' => undef, + @_ + ); + + $self->{list} = $args{list}; + $self->{server} = $args{server}; + $self->{port} = $args{port}; + $self->{timeout} = $args{timeout}; + $self->{useragent} = $args{useragent}; + $self->{_ua} = LWP::UserAgent->new(keep_alive => 1); + $self->{_ua}->agent($self->{useragent}); + $self->{_ua}->timeout($self->{timeout}); + + $self->{last_err} = undef; + return $self; +} + +sub expand { + my __PACKAGE__ $self = shift; + my $range = shift; + return $range unless $range; + my $query = 'expand'; + + $query = 'list' if ( $self->{list} ); + + $self->{last_err} = undef; + my $ua = $self->{_ua}; + my $server = $self->{server}; + my $port = $self->{port}; + my $req; + $req = HTTP::Request->new( + GET => "http://$server:$port/range/$query?" . uri_escape($range)); + my $res = $ua->request($req); + + my @nodes; + if ($res->is_success) { + if ( $self->{list} ) { + for my $line ( split ("\n",$res->content) ) { + chomp $line; + push @nodes,$line; + } + } + else { + @nodes = _simple_expand($res->content); + } + $self->{last_err} = $res->header('RangeException'); + } + else { + + # try again + my $res = $ua->request($req); + if ($res->is_success) { + if ( $self->{list} ) { + for my $line ( split ("\n",$res->content) ) { + chomp $line; + push @nodes,$line; + } + } + else { + @nodes = _simple_expand($res->content); + } + $self->{last_err} = $res->header('RangeException'); + } + else { + $self->{last_err} = $res->status_line; + } + } + return @nodes; +} + +sub last_err { + my __PACKAGE__ $self = shift; + return $self->{last_err}; +} + +sub _simple_compress { + my @nodes = @{ $_[0] }; + my %set; + @set{@nodes} = undef; + @nodes = keys %set; + @nodes = _sort_nodes(\@nodes); + + my @result; + my ($prev_prefix, $prev_digits, $prev_suffix) = ("", undef, ""); + my $prev_n; + my ($prefix, $digits, $suffix); + my $count = 0; + + for my $n (@nodes) { + if ($n =~ /^$node_regex$/) { + ($prefix, $digits, $suffix) = ($1, $2, $3); + $prefix = "" unless defined $prefix; + $suffix = "" unless defined $suffix; + } + else { + ($prefix, $digits, $suffix) = ($n, undef, undef); + } + if ( defined $digits + and $prefix eq $prev_prefix + and $suffix eq $prev_suffix + and defined $prev_digits + and $digits == $prev_digits + $count + 1) + { + $count++; + next; + } + if (defined $prev_n) { + if ($count > 0) { + push @result, + _get_group($prev_prefix, $prev_digits, $count, $prev_suffix); + } + else { + push @result, $prev_n; + } + } + + $prev_n = $n; + $prev_prefix = $prefix; + $prev_digits = $digits; + $prev_suffix = $suffix; + $count = 0; + } + + if ($count > 0) { + push @result, + _get_group($prev_prefix, $prev_digits, $count, $prev_suffix); + } + else { + push @result, $prev_n; + } + + return join(",", @result); +} + +sub _extra_compress { + my @nodes = @{ $_[0] }; + my %domains; + for (@nodes) { + s/^([a-z]+)(\d+)([a-z]\w+)\./$1$2.UNDOXXX$3./; + } + my $result = _simple_compress(\@nodes); + for ($result) { + s/(\d+-\d+)\.UNDOXXX/{$1}/g; + s/(\d+)\.UNDOXXX/$1/g; + } + return $result; +} + +sub compress { + my __PACKAGE__ $self = shift; + my $ref = ref $_[0]; + + $self->{last_err} = undef; + my @nodes = $ref ? @{ $_[0] } : @_; + unless (@nodes) { + $self->{last_err} = "No nodes specified."; + return; + } + + # do some extra work to achieve the extra compression + my %domain; + my @no_domains; + for my $node (@nodes) { + return _simple_compress(\@nodes) + if ($node =~ /^(?:"|q\()/); + + my ($host, $domain) = split '\.', $node, 2; + if ($domain) { + push @{ $domain{$domain} }, $node; + } + else { + push @no_domains, $host; + } + } + my @result; + if (@no_domains) { + push @result, _simple_compress(\@no_domains); + } + for my $domain (sort keys %domain) { + my $r = _extra_compress($domain{$domain}); + for ($r) { + s/\.$domain,/,/g; + s/\.$domain$//; + $_ = "{$_}" if /,/; + } + push @result, "$r.$domain"; + } + return join(",", @result); +} + +sub _sort_nodes { + my $ref_nodes = shift; + + my @sorted = + map { $_->[0] } + sort { + $a->[1] cmp $b->[1] + || $a->[3] cmp $b->[3] + || $a->[2] <=> $b->[2] + || $a->[0] cmp $b->[0] + } + map { + if (/\A$node_regex\z/) + { + [ + $_, + defined $1 ? $1 : "", + defined $2 ? $2 : 0, + defined $3 ? $3 : "" + ]; + } + else { [ $_, "", 0, "" ] } + } @$ref_nodes; + return @sorted; +} + +sub _simple_expand { + my $range_with_commas = shift; + return unless $range_with_commas; + + my @res; + my @parts = ($range_with_commas =~ /("[^"]+"),?/gs); + @parts = split(",", $range_with_commas) unless @parts; + + for my $range (@parts) { + if ( + $range =~ /\A + $node_regex + - # our separator is '-' + \1? # the prefix again, which is optional + (\d+) # and the end of the range + ((?(3) \3 | # if the domain matched before, we want it here + (?:\.[-A-Za-z\d.]+)?)) # if it didn't then we can have a new + # one here, like foo1-3.search + \z/xs + ) + { + my ($prefix, $start, $suf1, $end, $suf2) = ($1, $2, $3, $4, $5); + $prefix = "" unless defined $prefix; + my $suffix = ""; + if (defined $suf1 and defined $suf2) { + if ($suf1 ne $suf2) { + warn "Different suffixes: $suf1 $suf2"; + } + $suffix = $suf1; + } + elsif (defined $suf2) { + $suffix = $suf2; + } + my $len = length($start); + + # pad $end with leading characters from start so we can + # type 01-3 and expand that to 01,02,03 or maybe + # ks301000-9 for ks301000-301009 + my $len_end = length($end); + $end = substr($start, 0, $len - $len_end) . $end + if $len_end < $len; + push @res, + map { $_ = sprintf("$prefix%0*d$suffix", $len, $_) } + ($start .. $end); + } + else { + + # single machine + push @res, $range; + } + } + return @res; +} + +# compress_range related stuff +sub _ignore_common_prefix { + my ($start, $end) = @_; + + my $len_start = length $start; + if ($len_start < length $end) { + return $end; + } + + my $i; + for ($i = 0 ; $i < $len_start ; $i++) { + last if substr($start, $i, 1) ne substr($end, $i, 1); + } + return substr($end, $i); +} + +sub _get_group { + my ($prefix, $digits, $count, $suffix) = @_; + + $prefix = "" unless defined $prefix; + my $group = sprintf("%s%0*d-%s", + $prefix, length($digits), $digits, + _ignore_common_prefix($digits, $digits + $count)); + + $suffix = "" unless defined $suffix; + return $group . $suffix; +} + +1; + +__END__ + +=head1 NAME + +Seco::Data::Range - OO interface to ranged + +=head1 SYNOPSIS + + use Seco::Data::Range; + + my $range = Seco::Data::Range->new; + or + my $range = Seco::Data::Range->new(list => '1' ); + + This will use "list" instead of "expand" and works as expected for any + range expressions using q(). Only use when needed. + + my @nodes = $range->expand("vlan(pain)"); + my $nodes = $range->compress(["foo10", "foo11"]); + my $err = $range->last_err; + +=head1 DESCRIPTION + +TODO + +=head2 EXPORT + +None by default. + +=head1 SEE ALSO + +ranged, range functions. + +=head1 AUTHOR + +Daniel Muino, Edmuino@yahoo-inc.comE + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2006 Yahoo! Inc. + +=cut diff --git a/perl_seco_data_range/source/lib/Seco/Data/Range/Simple.pm b/perl_seco_data_range/source/lib/Seco/Data/Range/Simple.pm new file mode 100644 index 0000000..7481bca --- /dev/null +++ b/perl_seco_data_range/source/lib/Seco/Data/Range/Simple.pm @@ -0,0 +1,45 @@ +package Seco::Data::Range::Simple; +=head1 NAME + + Seco::Data::Range::Simple + +=head1 + + Not meant to be used directly. This super simple stub is for loading + into other bootstrap software such as a ybiip profile where + the environment is not fully set up (no lwp) + + usage: simple_range("rangehost.yahoo.com", 9999, '@ALL'); + +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +=cut + + +sub simple_range { + my ($rangehost, $rangeport, $rangequery, $timeout) = @_; + $timeout ||= 15; + my $s = IO::Socket::INET->new( + PeerHost => $rangehost, + PeerPort => $rangeport, + Proto => "tcp", + Timeout => $timeout, + ); + syswrite $s, "GET /range/list?$rangequery\n"; + my @names = <$s>; + chomp for @names; + return @names; +} + +1; + +=head1 AUTHOR + +Evan Miller, Eeam@yahoo-inc.comE + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2006 Yahoo! Inc. + +=cut diff --git a/perl_seco_data_range/source/t/Seco-Data-Range.t b/perl_seco_data_range/source/t/Seco-Data-Range.t new file mode 100644 index 0000000..ef04245 --- /dev/null +++ b/perl_seco_data_range/source/t/Seco-Data-Range.t @@ -0,0 +1,15 @@ +# Before `make install' is performed this script should be runnable with +# `make test'. After `make install' it should work as `perl Seco-Data-Range.t' + +######################### + +# change 'tests => 1' to 'tests => last_test_to_print'; + +use Test::More tests => 1; +BEGIN { use_ok('Seco::Data::Range') }; + +######################### + +# Insert your test code below, the Test::More module is use()ed here so read +# its man page ( perldoc Test::More ) for help writing this test script. + diff --git a/perl_seco_libcrange/index.yaml b/perl_seco_libcrange/index.yaml new file mode 100644 index 0000000..cb2f0a4 --- /dev/null +++ b/perl_seco_libcrange/index.yaml @@ -0,0 +1,4 @@ +default: + name: perl-seco-libcrange + version: '1.0' + summary: Seco::Libcrange diff --git a/perl_seco_libcrange/source/Changes b/perl_seco_libcrange/source/Changes new file mode 100644 index 0000000..bbc4a12 --- /dev/null +++ b/perl_seco_libcrange/source/Changes @@ -0,0 +1,6 @@ +Revision history for Perl extension Seco::Libcrange. + +0.01 Thu Feb 25 02:35:34 2010 + - original version; created by h2xs 1.23 with options + -A -n Seco::Libcrange + diff --git a/perl_seco_libcrange/source/Libcrange.xs b/perl_seco_libcrange/source/Libcrange.xs new file mode 100644 index 0000000..104da65 --- /dev/null +++ b/perl_seco_libcrange/source/Libcrange.xs @@ -0,0 +1,128 @@ +/* +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + */ + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#include + +static libcrange* +__get_libcrange_ptr( SV *var ) +{ + if (!(sv_isobject((SV*)var) && sv_derived_from((SV*)var,"Seco::Libcrange"))) + return 0; + HV* h = (HV *)SvRV( var ); + SV** lr = hv_fetch(h,"__libcrange_ptr", strlen("__libcrange_ptr"),0); + if(!lr) + return 0; + return INT2PTR(libcrange*, SvIV(*lr)); +} + +MODULE = Seco::Libcrange PACKAGE = Seco::Libcrange +PROTOTYPES: ENABLE +BOOT: + apr_initialize(); + +SV* +new(class,config) + char* class; + char* config; + + INIT: + SV* iv; + HV* myself; + HV* stash; + + CODE: + apr_pool_t* lr_pool = NULL; + apr_pool_create(&lr_pool,NULL); + libcrange *lcr = libcrange_new(lr_pool,config); + myself = newHV(); + iv = newSViv(PTR2IV(lcr)); + hv_store( myself, "__libcrange_ptr",strlen("__libcrange_ptr"), iv, 0 ); + /* Return a blessed reference to the HV */ + RETVAL = newRV_noinc( (SV *)myself ); + stash = gv_stashpv( class, TRUE ); + sv_bless( (SV *)RETVAL, stash ); + OUTPUT: + RETVAL + + +void +expand(self,range) + SV* self + char* range + INIT: + libcrange *lr; + + PPCODE: + lr = __get_libcrange_ptr(self); + if( lr == NULL || range == NULL) + XSRETURN_UNDEF; + apr_pool_t* req_pool = NULL; + apr_pool_create(&req_pool,NULL); + range_request *rr = range_expand(lr,req_pool,range); + if (rr == NULL) + { + apr_pool_destroy(req_pool); + XSRETURN_UNDEF; + } + if ( range_request_has_warnings(rr) + && + SvTRUE(get_sv("Seco::Libcrange::raise_exceptions",TRUE))) + { + apr_pool_destroy(req_pool); + croak("%s", range_request_warnings(rr)); + } + const char **nodes = range_request_nodes(rr); + if (nodes == NULL) + { + apr_pool_destroy(req_pool); + XSRETURN_UNDEF; + } + while (*nodes) + { + XPUSHs(sv_2mortal(newSVpv(*nodes++, 0))); + } + apr_pool_destroy(req_pool); + + +SV* +compress_xs(self,nodes) + SV* self + SV* nodes + INIT: + const char ** node_lst; + char *range; + I32 num_nodes; + int i; + SV *ret; + libcrange *lr; + + CODE: + lr = __get_libcrange_ptr(self); + if( lr == NULL) + XSRETURN_UNDEF; + if ((!SvROK(nodes)) || (SvTYPE(SvRV(nodes)) != SVt_PVAV) + || ((num_nodes = av_len((AV *)SvRV(nodes))) < 0)) + XSRETURN_UNDEF; + node_lst = malloc(sizeof(char*) * (num_nodes + 2)); + apr_pool_t* req_pool = NULL; + apr_pool_create(&req_pool,NULL); + for (i=0; i<=num_nodes; i++) { + char* node = SvPV_nolen(*av_fetch((AV *)SvRV(nodes), i, 0)); + node_lst[i] = node; + } + node_lst[num_nodes + 1] = NULL; + range = range_compress(lr,req_pool,node_lst); + free(node_lst); + apr_pool_destroy(req_pool); + if (NULL == range) + XSRETURN_UNDEF; + RETVAL = newSVpv(range, 0); + /* free(range); */ + OUTPUT: + RETVAL diff --git a/perl_seco_libcrange/source/MANIFEST b/perl_seco_libcrange/source/MANIFEST new file mode 100644 index 0000000..ed15d6a --- /dev/null +++ b/perl_seco_libcrange/source/MANIFEST @@ -0,0 +1,8 @@ +Changes +Libcrange.xs +Makefile.PL +MANIFEST +ppport.h +README +t/Seco-Libcrange.t +lib/Seco/Libcrange.pm diff --git a/perl_seco_libcrange/source/Makefile.PL b/perl_seco_libcrange/source/Makefile.PL new file mode 100644 index 0000000..480899b --- /dev/null +++ b/perl_seco_libcrange/source/Makefile.PL @@ -0,0 +1,34 @@ +use 5.006; +use ExtUtils::MakeMaker; + +sub find_executable { + my $exe = shift; + my @PATH = split ":", $ENV{PATH}; + my $dir = (grep { -x "$_/$exe" } @PATH)[0]; + return unless $dir; + + print "Using $dir/$exe\n"; + return "$dir/$exe"; +} + +my $apr_config = find_executable("apr-config"); +$apr_config ||= find_executable("apr-1-config"); + +my $ld_flags = `$apr_config --link-ld`; +chomp($ld_flags); +my $c_flags = `$apr_config --cflags --cppflags --includes`; +chomp($c_flags); + +WriteMakefile( + NAME => 'Seco::Libcrange', + VERSION_FROM => 'lib/Seco/Libcrange.pm', # finds $VERSION + PREREQ_PM => {}, # e.g., Module::Name => 1.1 + ($] >= 5.005 ? ## Add these new keywords supported since 5.005 + (ABSTRACT_FROM => 'lib/Seco/Libcrange.pm', # retrieve abstract from module + AUTHOR => 'Syam Puranam ') : ()), + LIBS => ["-L/usr/local/lib -lcrange $ld_flags"], # e.g., '-lm' + DEFINE => '', # e.g., '-DHAVE_SOMETHING' + INC => "-I. $c_flags", # e.g., '-I. -I/usr/include/other' + # Un-comment this if you add C files to link with later: + # OBJECT => '$(O_FILES)', # link all the C files too +); diff --git a/perl_seco_libcrange/source/README b/perl_seco_libcrange/source/README new file mode 100644 index 0000000..fc74f7e --- /dev/null +++ b/perl_seco_libcrange/source/README @@ -0,0 +1,40 @@ +Seco-Libcrange version 0.01 +=========================== + +The README is used to introduce the module and provide instructions on +how to install the module, any machine dependencies it may have (for +example C compilers and installed libraries) and any other information +that should be provided before the module is installed. + +A README file is required for CPAN modules since CPAN extracts the +README file from a module distribution so that people browsing the +archive can use it get an idea of the modules uses. It is usually a +good idea to provide version information here so that people can +decide whether fixes for the module are worth downloading. + +INSTALLATION + +To install this module type the following: + + perl Makefile.PL + make + make test + make install + +DEPENDENCIES + +This module requires these other modules and libraries: + + blah blah blah + +COPYRIGHT AND LICENCE + +Put the correct copyright and licence information here. + +Copyright (C) 2010 by Yahoo! Inc. + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself, either Perl version 5.8.5 or, +at your option, any later version of Perl 5 you may have available. + + diff --git a/perl_seco_libcrange/source/lib/Seco/Libcrange.pm b/perl_seco_libcrange/source/lib/Seco/Libcrange.pm new file mode 100644 index 0000000..cd50adc --- /dev/null +++ b/perl_seco_libcrange/source/lib/Seco/Libcrange.pm @@ -0,0 +1,133 @@ +package Seco::Libcrange; + +# Copyright (c) 2011, Yahoo! Inc. All rights reserved. +# Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +use 5.008005; +use strict; +use warnings; + +our $VERSION = '0.01'; +require XSLoader; +XSLoader::load('Seco::Libcrange', $VERSION); + +# Preloaded methods go here. +sub _extra_compress { + my $self = shift; + my @nodes = @{ $_[0] }; + my %domains; + for (@nodes) { + s/^([a-z]+)(\d+)([a-z]\w+)\./$1$2.UNDOXXX$3./; + } + my $result = $self->compress_xs(\@nodes); + for ($result) { + s/(\d+-\d+)\.UNDOXXX/{$1}/g; + s/(\d+)\.UNDOXXX/$1/g; + } + return $result; +} + +sub compress { + my $self = shift; + return unless (defined $_[0]); + my $ref = ref $_[0]; + my @nodes = $ref ? @{ $_[0] } : @_; + return unless (@nodes); + ### UGLY SDR compat + my %domain; + my @no_domains; + foreach my $node (@nodes) { + ## spare literal's from extra compression + return $self->compress_xs(\@nodes) + if ($node =~ /^(?:"|q\()/); + my ($host, $domain) = split('\.',$node,2); + if ($domain) { + push @{ $domain{$domain} }, $node; + } + else { + push @no_domains, $host; + } + } + my @result; + if (@no_domains) { + push @result, $self->compress_xs(\@no_domains); + } + for my $domain (sort keys %domain) { + my $r = $self->_extra_compress($domain{$domain}); + for ($r) { + s/\.$domain,/,/g; + s/\.$domain$//; + $_ = "{$_}" if /,/; + } + push @result, "$r.$domain"; + } + return join(",", @result); +} + +sub get_range { + my $r = shift; + return unless ($_[0]); + return $r->compress($r->expand($_[0])); +} + +sub range_sub { + my $r = shift; + my ( $range1, $range2 ) = @_; + return $range1 unless ( defined $range2 ); + return $r->get_range( $range1 . ",-(" . $range2 . ")" ); +} + +sub range_add { + my $r = shift; + my ( $range1, $range2 ) = @_; + return $range1 unless ( defined $range2 ); + return $r->get_range( $range1 . ",(" . $range2 . ")" ); +} + +sub range_and { + my $r = shift; + my ( $range1, $range2 ) = @_; + return $r->get_range( $range1 . ",&(" . $range2 . ")" ); +} + +sub is_simple_range { + my $r = shift; + my $range = shift; + return 1 if ( $range =~ /^["\s\.\w,\-\{\}]*$/ ); + return 0; +} + +1; +__END__ + +=head1 NAME + +Seco::Libcrange - Perl extension for Libcrange + +=head1 SYNOPSIS + + use Seco::Libcrange; + my $r = Seco::Libcrange::->new('/etc/libcrange.conf'); + my @keys = $r->expand('%ngd-ac4-rmxad2p0:KEYS') + my $range = $r->compress(\@keys); + or + my $range = $r->compress(@keys); + +=head1 DESCRIPTION + + See SYNOPSIS + + +=head1 AUTHOR + +Syam Puranam, Esyam@yahoo-inc.com + +=head1 COPYRIGHT AND LICENSE + +Copyright (C) 2010 by Yahoo! + +=cut + +=head1 BUGS + +Currently only one instance of Libcrange works :-\ diff --git a/perl_seco_libcrange/source/ppport.h b/perl_seco_libcrange/source/ppport.h new file mode 100644 index 0000000..3760837 --- /dev/null +++ b/perl_seco_libcrange/source/ppport.h @@ -0,0 +1,1096 @@ + +/* ppport.h -- Perl/Pollution/Portability Version 2.011 + * + * Automatically Created by Devel::PPPort on Thu Feb 25 02:35:34 2010 + * + * Do NOT edit this file directly! -- Edit PPPort.pm instead. + * + * Version 2.x, Copyright (C) 2001, Paul Marquess. + * Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + * This code may be used and distributed under the same license as any + * version of Perl. + * + * This version of ppport.h is designed to support operation with Perl + * installations back to 5.004, and has been tested up to 5.8.1. + * + * If this version of ppport.h is failing during the compilation of this + * module, please check if a newer version of Devel::PPPort is available + * on CPAN before sending a bug report. + * + * If you are using the latest version of Devel::PPPort and it is failing + * during compilation of this module, please send a report to perlbug@perl.com + * + * Include all following information: + * + * 1. The complete output from running "perl -V" + * + * 2. This file. + * + * 3. The name & version of the module you were trying to build. + * + * 4. A full log of the build that failed. + * + * 5. Any other information that you think could be relevant. + * + * + * For the latest version of this code, please retreive the Devel::PPPort + * module from CPAN. + * + */ + +/* + * In order for a Perl extension module to be as portable as possible + * across differing versions of Perl itself, certain steps need to be taken. + * Including this header is the first major one, then using dTHR is all the + * appropriate places and using a PL_ prefix to refer to global Perl + * variables is the second. + * + */ + + +/* If you use one of a few functions that were not present in earlier + * versions of Perl, please add a define before the inclusion of ppport.h + * for a static include, or use the GLOBAL request in a single module to + * produce a global definition that can be referenced from the other + * modules. + * + * Function: Static define: Extern define: + * newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + * + */ + + +/* To verify whether ppport.h is needed for your module, and whether any + * special defines should be used, ppport.h can be run through Perl to check + * your source code. Simply say: + * + * perl -x ppport.h *.c *.h *.xs foo/bar*.c [etc] + * + * The result will be a list of patches suggesting changes that should at + * least be acceptable, if not necessarily the most efficient solution, or a + * fix for all possible problems. It won't catch where dTHR is needed, and + * doesn't attempt to account for global macro or function definitions, + * nested includes, typemaps, etc. + * + * In order to test for the need of dTHR, please try your module under a + * recent version of Perl that has threading compiled-in. + * + */ + + +/* +#!/usr/bin/perl +@ARGV = ("*.xs") if !@ARGV; +%badmacros = %funcs = %macros = (); $replace = 0; +foreach () { + $funcs{$1} = 1 if /Provide:\s+(\S+)/; + $macros{$1} = 1 if /^#\s*define\s+([a-zA-Z0-9_]+)/; + $replace = $1 if /Replace:\s+(\d+)/; + $badmacros{$2}=$1 if $replace and /^#\s*define\s+([a-zA-Z0-9_]+).*?\s+([a-zA-Z0-9_]+)/; + $badmacros{$1}=$2 if /Replace (\S+) with (\S+)/; +} +foreach $filename (map(glob($_),@ARGV)) { + unless (open(IN, "<$filename")) { + warn "Unable to read from $file: $!\n"; + next; + } + print "Scanning $filename...\n"; + $c = ""; while () { $c .= $_; } close(IN); + $need_include = 0; %add_func = (); $changes = 0; + $has_include = ($c =~ /#.*include.*ppport/m); + + foreach $func (keys %funcs) { + if ($c =~ /#.*define.*\bNEED_$func(_GLOBAL)?\b/m) { + if ($c !~ /\b$func\b/m) { + print "If $func isn't needed, you don't need to request it.\n" if + $changes += ($c =~ s/^.*#.*define.*\bNEED_$func\b.*\n//m); + } else { + print "Uses $func\n"; + $need_include = 1; + } + } else { + if ($c =~ /\b$func\b/m) { + $add_func{$func} =1 ; + print "Uses $func\n"; + $need_include = 1; + } + } + } + + if (not $need_include) { + foreach $macro (keys %macros) { + if ($c =~ /\b$macro\b/m) { + print "Uses $macro\n"; + $need_include = 1; + } + } + } + + foreach $badmacro (keys %badmacros) { + if ($c =~ /\b$badmacro\b/m) { + $changes += ($c =~ s/\b$badmacro\b/$badmacros{$badmacro}/gm); + print "Uses $badmacros{$badmacro} (instead of $badmacro)\n"; + $need_include = 1; + } + } + + if (scalar(keys %add_func) or $need_include != $has_include) { + if (!$has_include) { + $inc = join('',map("#define NEED_$_\n", sort keys %add_func)). + "#include \"ppport.h\"\n"; + $c = "$inc$c" unless $c =~ s/#.*include.*XSUB.*\n/$&$inc/m; + } elsif (keys %add_func) { + $inc = join('',map("#define NEED_$_\n", sort keys %add_func)); + $c = "$inc$c" unless $c =~ s/^.*#.*include.*ppport.*$/$inc$&/m; + } + if (!$need_include) { + print "Doesn't seem to need ppport.h.\n"; + $c =~ s/^.*#.*include.*ppport.*\n//m; + } + $changes++; + } + + if ($changes) { + open(OUT,">ppport.h.$$"); + print OUT $c; + close(OUT); + open(DIFF, "diff -u $filename ppport.h.$$|"); + while () { s!ppport\.h\.$$!$filename.patched!; print STDOUT; } + close(DIFF); + unlink("ppport.h.$$"); + } else { + print "Looks OK\n"; + } +} +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef PERL_REVISION +# ifndef __PATCHLEVEL_H_INCLUDED__ +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define PERL_BCDVERSION ((PERL_REVISION * 0x1000000L) + (PERL_VERSION * 0x1000L) + PERL_SUBVERSION) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ + +#ifndef ERRSV +# define ERRSV perl_get_sv("@",FALSE) +#endif + +#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION <= 5)) +/* Replace: 1 */ +# define PL_Sv Sv +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_defgv defgv +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_hints hints +# define PL_na na +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfpv rsfp +# define PL_stdingv stdingv +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +/* Replace: 0 */ +#endif + +#ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +#else +# define PERL_UNUSED_DECL +#endif + +#ifndef dNOOP +# define NOOP (void)0 +# define dNOOP extern int Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef dTHR +# define dTHR dNOOP +#endif + +#ifndef dTHX +# define dTHX dNOOP +# define dTHXa(x) dNOOP +# define dTHXoa(x) dNOOP +#endif + +#ifndef pTHX +# define pTHX void +# define pTHX_ +# define aTHX +# define aTHX_ +#endif + +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif + +/* IV could also be a quad (say, a long long), but Perls + * capable of those should have IVSIZE already. */ +#if !defined(IVSIZE) && defined(LONGSIZE) +# define IVSIZE LONGSIZE +#endif +#ifndef IVSIZE +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR + +#if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +#else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +#endif +#define NUM2PTR(any,d) (any)(PTRV)(d) +#define PTR2IV(p) INT2PTR(IV,p) +#define PTR2UV(p) INT2PTR(UV,p) +#define PTR2NV(p) NUM2PTR(NV,p) +#if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +#else +# define PTR2ul(p) INT2PTR(unsigned long,p) +#endif + +#endif /* !INT2PTR */ + +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,flags) gv_stashpv(str,flags) +#endif + +#ifndef newSVpvn +# define newSVpvn(data,len) ((len) ? newSVpv ((data), (len)) : newSVpv ("", 0)) +#endif + +#ifndef newRV_inc +/* Replace: 1 */ +# define newRV_inc(sv) newRV(sv) +/* Replace: 0 */ +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef newRV_noinc +# ifdef __GNUC__ +# define newRV_noinc(sv) \ + ({ \ + SV *nsv = (SV*)newRV(sv); \ + SvREFCNT_dec(sv); \ + nsv; \ + }) +# else +# if defined(USE_THREADS) +static SV * newRV_noinc (SV * sv) +{ + SV *nsv = (SV*)newRV(sv); + SvREFCNT_dec(sv); + return nsv; +} +# else +# define newRV_noinc(sv) \ + (PL_Sv=(SV*)newRV(sv), SvREFCNT_dec(sv), (SV*)PL_Sv) +# endif +# endif +#endif + +/* Provide: newCONSTSUB */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION < 63)) + +#if defined(NEED_newCONSTSUB) +static +#else +extern void newCONSTSUB(HV * stash, char * name, SV *sv); +#endif + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) +void +newCONSTSUB(stash,name,sv) +HV *stash; +char *name; +SV *sv; +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_VERSION < 3) || ((PERL_VERSION == 3) && (PERL_SUBVERSION < 22)) + /* before 5.003_22 */ + start_subparse(), +#else +# if (PERL_VERSION == 3) && (PERL_SUBVERSION == 22) + /* 5.003_22 */ + start_subparse(0), +# else + /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +# endif +#endif + + newSVOP(OP_CONST, 0, newSVpv(name,0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif + +#endif /* newCONSTSUB */ + +#ifndef START_MY_CXT + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_VERSION < 4 || (PERL_VERSION == 4 && PERL_SUBVERSION < 68 )) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = perl_get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#else /* single interpreter */ + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif + +#endif /* START_MY_CXT */ + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef AvFILLp /* Older perls (<=5.003) lack AvFILLp */ +# define AvFILLp AvFILL +#endif + +#ifdef SvPVbyte +# if PERL_REVISION == 5 && PERL_VERSION < 7 + /* SvPVbyte does not work in perl-5.6.1, borrowed version for 5.7.3 */ +# undef SvPVbyte +# define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : my_sv_2pvbyte(aTHX_ sv, &lp)) + static char * + my_sv_2pvbyte(pTHX_ register SV *sv, STRLEN *lp) + { + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); + } +# endif +#else +# define SvPVbyte SvPV +#endif + +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_nolen(sv)) + static char * + sv_2pv_nolen(pTHX_ register SV *sv) + { + STRLEN n_a; + return sv_2pv(sv, &n_a); + } +#endif + +#ifndef get_cv +# define get_cv(name,create) perl_get_cv(name,create) +#endif + +#ifndef get_sv +# define get_sv(name,create) perl_get_sv(name,create) +#endif + +#ifndef get_av +# define get_av(name,create) perl_get_av(name,create) +#endif + +#ifndef get_hv +# define get_hv(name,create) perl_get_hv(name,create) +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef eval_pv +# define eval_pv perl_eval_pv +#endif + +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#if (PERL_VERSION > 6) || ((PERL_VERSION == 6) && (PERL_SUBVERSION >= 1)) +#define I32_CAST +#else +#define I32_CAST (I32*) +#endif + +#ifndef grok_hex +static UV _grok_hex (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_hex(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_hex(string, len, flags, result) \ + _grok_hex((string), (len), (flags), (result)) +#endif + +#ifndef grok_oct +static UV _grok_oct (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_oct(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_oct(string, len, flags, result) \ + _grok_oct((string), (len), (flags), (result)) +#endif + +#if !defined(grok_bin) && defined(scan_bin) +static UV _grok_bin (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_bin(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_bin(string, len, flags, result) \ + _grok_bin((string), (len), (flags), (result)) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE \ + (PL_curcop == &PL_compiling ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + + +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +# define IS_NUMBER_NOT_INT 0x04 +# define IS_NUMBER_NEG 0x08 +# define IS_NUMBER_INFINITY 0x10 +# define IS_NUMBER_NAN 0x20 +#endif + +#ifndef grok_numeric_radix +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(aTHX_ sp, send) + +#define grok_numeric_radix Perl_grok_numeric_radix + +bool +Perl_grok_numeric_radix(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#if (PERL_VERSION > 6) || ((PERL_VERSION == 6) && (PERL_SUBVERSION >= 1)) + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* pre5.6.0 perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h */ +#include + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif /* PERL_VERSION */ +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif /* grok_numeric_radix */ + +#ifndef grok_number + +#define grok_number Perl_grok_number + +int +Perl_grok_number(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif /* grok_number */ + +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/perl_seco_libcrange/source/t/Seco-Libcrange.t b/perl_seco_libcrange/source/t/Seco-Libcrange.t new file mode 100644 index 0000000..d8242a3 --- /dev/null +++ b/perl_seco_libcrange/source/t/Seco-Libcrange.t @@ -0,0 +1,41 @@ +# Before `make install' is performed this script should be runnable with +# `make test'. After `make install' it should work as `perl Seco-Libcrange.t' + +######################### + +# change 'tests => 13' to 'tests => last_test_to_print'; + +use Test::More tests => 13; +BEGIN { use_ok('Seco::Libcrange') }; + +######################### + +# Insert your test code below, the Test::More module is use()ed here so read +# its man page ( perldoc Test::More ) for help writing this test script. + +my $r = Seco::Libcrange::->new('/etc/libcrange.conf'); + +ok (compare_arrays([$r->expand('1-2')], ['1','2']), '$r->expand'); +ok ($r->compress(['1','2']) eq '1-2', '$r->compress(\@)'); +ok ($r->compress(('1','2')) eq '1-2', '$r->compress(@)'); +ok ($r->compress(('"1.com"','"2.com"')) eq '"1.com","2.com"', '$r->compress(literals)'); +ok( !(defined $r->compress(undef)), '$r->compress(undef) returns undef'); +ok( !(defined $r->compress([])), '$r->compress([]) returns undef'); +ok( !defined $r->expand('1&2'), '$r->expand("1&2") returns undef'); +ok( !defined $r->expand(undef), '$r->expand(undef) returns undef'); +ok('1-2' eq $r->compress($r->expand('1-2')), '$r->compress($r->expand())'); +ok('"1.com","2.com"' eq $r->compress($r->expand('"1.com","2.com"')), '$r->compress($r->expand())'); +ok('"1.com","2.com"' eq $r->compress($r->expand('q(1.com),q(2.com)')), '$r->compress($r->expand())'); +ok('q(1.com),q(2.com)' eq $r->compress('q(1.com),q(2.com)'),'$r->compress()'); + + +sub compare_arrays { + my ($first, $second) = @_; + no warnings; # silence spurious undef complaints + return 0 unless @$first == @$second; + for (my $i = 0; $i < @$first; $i++) { + return 0 if $first->[$i] ne $second->[$i]; + } + return 1; +} +__END__ diff --git a/seco_awesomerange/index.yaml b/seco_awesomerange/index.yaml new file mode 100644 index 0000000..1e0c2da --- /dev/null +++ b/seco_awesomerange/index.yaml @@ -0,0 +1,98 @@ +--- +default: + name: fatperl-seco-awesomerange + summary: awesome test replacement for Seco::Range + version: '1.0.3' +# +# These will replace "perl-awesomerange" and "perl-awesomerange64" +# + +# Define what needs to be built for each platform. +# These will later be called like platforms. Arch types +# are appended so that we can create unique options on each. + +yss.i386.redhat: + builds: + - perl-seco-awesomerange.i386.redhat +yst.i386.redhat: + builds: + - perl-seco-awesomerange.i386.redhat + - fixperl-seco-awesomerange.i386.redhat +yss.x86_64.redhat: + builds: + - perl-seco-awesomerange.x86_64.redhat +yst.x86_64.redhat: + builds: + - perl-seco-awesomerange.x86_64.redhat + - fixperl-seco-awesomerange.x86_64.redhat +i386.debian: + builds: + - perl-seco-awesomerange.i386.debian + - fixperl-seco-awesomerange.i386.debian + + +# Specific notes now for each build. Total of 4 builds +# (x86 fixperl should refuse to build, suggesting the i386 build be used) + + +###################################################### +# DEBIAN # +###################################################### + +fixperl-seco-awesomerange.i386.debian: + name: fixperl-seco-awesomerange + perl: /usr/local/bin/perl + requires: + - librange + - perl + provides: + - perl-awesomerange + obsoletes: + - perl-awesomerange + +perl-seco-awesomerange.i386.debian: + meta: 1 + name: perl-seco-awesomerange + + +###################################################### +# REDHAT 32 BIT # +###################################################### + +fixperl-seco-awesomerange.i386.redhat: + perl: /usr/local/bin/perl + name: fixperl-seco-awesomerange + obsoletes: + - perl-awesomerange + provides: + - perl-awesomerange + requires: + - librange + - perl + +perl-seco-awesomerange.i386.redhat: + perl: /usr/bin/perl + name: perl-seco-awesomerange + requires: + - librange + - perl + +###################################################### +# REDHAT 64 BIT # +###################################################### + +fixperl-seco-awesomerange.x86_64.redhat: + name: fixperl-seco-awesomerange + preferarch: i386 + +perl-seco-awesomerange.x86_64.redhat: + perl: /usr/bin/perl + name: perl-seco-awesomerange + obsoletes: + - perl-awesomerange64 + provides: + - perl-awesomerange64 + requires: + - librange64 + - perl + diff --git a/seco_awesomerange/scripts/build b/seco_awesomerange/scripts/build new file mode 100755 index 0000000..caa7fcb --- /dev/null +++ b/seco_awesomerange/scripts/build @@ -0,0 +1,5 @@ +#!/bin/sh + +# Perl wants to rewrite these files. +pwd | grep '/tmp/' && chmod u+w const-*.inc +exec /usr/share/multipkg/scripts/build diff --git a/seco_awesomerange/source/AwesomeRange.xs b/seco_awesomerange/source/AwesomeRange.xs new file mode 100644 index 0000000..00175a3 --- /dev/null +++ b/seco_awesomerange/source/AwesomeRange.xs @@ -0,0 +1,162 @@ +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" + +#include "range.h" + +/* + +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +*/ + +MODULE = Seco::AwesomeRange PACKAGE = Seco::AwesomeRange + +BOOT: + range_startup(); + sv_setsv( get_sv("Seco::AwesomeRange::librange_version", TRUE), + newSVpv(LIBRANGE_VERSION, 0) + ); + +void +range_set_altpath(path) + char* path + CODE: + range_clear_exception(); + range_set_altpath(path); + if (range_get_exception() && SvTRUE( get_sv("Seco::AwesomeRange::raise_exceptions", TRUE) )) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), range_get_exception() ); + + +void +range_expand(range) + char* range + INIT: + const char** res; + int i; + PPCODE: + range_clear_exception(); + res = range_expand(range); + if (range_get_exception() && SvTRUE( get_sv("Seco::AwesomeRange::raise_exceptions", TRUE) )) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + if (NULL == res) + XSRETURN_UNDEF; + for (i=0; res[i] != NULL; i++) + XPUSHs(sv_2mortal(newSVpv(res[i], 0))); + range_free_nodes(res); + +void +range_expand_sorted(range) + char* range + INIT: + const char ** res; + int i; + PPCODE: + range_clear_exception(); + res = range_expand_sorted(range); + if (range_get_exception() && SvTRUE( get_sv("Seco::AwesomeRange::raise_exceptions", TRUE) )) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + if (NULL == res) + XSRETURN_UNDEF; + for (i=0; res[i] != NULL; i++) + XPUSHs(sv_2mortal(newSVpv(res[i], 0))); + range_free_nodes(res); + +SV * +range_compress_xs(nodes, separator) + SV* nodes + char* separator + INIT: + const char ** node_lst; + char *range; + I32 num_nodes; + int i; + SV *ret; +/* only accept arrayref. In perl, ref any list sent */ +/* FIXME make throw exception or something better if Seco::AwesomeRange::raise_exceptions on */ + if ((!SvROK(nodes)) || (SvTYPE(SvRV(nodes)) != SVt_PVAV) + || ((num_nodes = av_len((AV *)SvRV(nodes))) < 0)) + XSRETURN_UNDEF; + node_lst = malloc(sizeof(char*) * (num_nodes + 2)); + CODE: + for (i=0; i<=num_nodes; i++) { + + char* node = SvPV_nolen(*av_fetch((AV *)SvRV(nodes), i, 0)); + node_lst[i] = node; + } + node_lst[num_nodes + 1] = NULL; + range_clear_exception(); + range = range_compress(node_lst, separator); + free(node_lst); + if (range_get_exception() && SvTRUE( get_sv("Seco::AwesomeRange::raise_exceptions", TRUE) )) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + if (NULL == range) + XSRETURN_UNDEF; + RETVAL = newSVpv(range, 0); + free(range); + OUTPUT: + RETVAL + +void +range_clear_caches() + CODE: + range_clear_exception(); + range_clear_caches(); + if (range_get_exception() && SvTRUE(get_sv("Seco::AwesomeRange::raise_exceptions", TRUE))) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + + + +void +range_want_caching(want) + int want; + CODE: + range_clear_exception(); + range_want_caching(want); + if (range_get_exception() && SvTRUE(get_sv("Seco::AwesomeRange::raise_exceptions", TRUE))) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + +void +range_want_warnings(want) + int want; + CODE: + range_clear_exception(); + range_want_warnings(want); + if (range_get_exception() && SvTRUE(get_sv("Seco::AwesomeRange::raise_exceptions", TRUE))) { + croak("%s", range_get_exception()); + } + sv_setpv( get_sv("Seco::AwesomeRange::errno", TRUE), + range_get_exception() + ); + +SV * +range_get_version() + CODE: + RETVAL = newSVpv(range_get_version(), 0); + OUTPUT: + RETVAL + diff --git a/seco_awesomerange/source/Changes b/seco_awesomerange/source/Changes new file mode 100644 index 0000000..34bdc08 --- /dev/null +++ b/seco_awesomerange/source/Changes @@ -0,0 +1,6 @@ +Revision history for Perl extension Seco::AwesomeRange. + +0.01 Thu Jul 7 04:44:41 2005 + - original version; created by h2xs 1.23 with options + -n Seco::AwesomeRange + diff --git a/seco_awesomerange/source/MANIFEST b/seco_awesomerange/source/MANIFEST new file mode 100644 index 0000000..82b4fe1 --- /dev/null +++ b/seco_awesomerange/source/MANIFEST @@ -0,0 +1,10 @@ +AwesomeRange.xs +Changes +Makefile.PL +MANIFEST +ppport.h +README +t/Seco-AwesomeRange.t +fallback/const-c.inc +fallback/const-xs.inc +lib/Seco/AwesomeRange.pm diff --git a/seco_awesomerange/source/Makefile b/seco_awesomerange/source/Makefile new file mode 100644 index 0000000..13d4909 --- /dev/null +++ b/seco_awesomerange/source/Makefile @@ -0,0 +1,878 @@ +# This Makefile is for the Seco::AwesomeRange extension to perl. +# +# It was generated automatically by MakeMaker version +# 6.17 (Revision: 1.133) from the contents of +# Makefile.PL. Don't edit this file, edit Makefile.PL instead. +# +# ANY CHANGES MADE HERE WILL BE LOST! +# +# MakeMaker ARGV: () +# +# MakeMaker Parameters: + +# ABSTRACT_FROM => q[lib/Seco/AwesomeRange.pm] +# AUTHOR => q[Evan Miller ] +# DEFINE => q[] +# INC => q[-I.] +# LIBS => [q[-lrange -lpcre -lm]] +# NAME => q[Seco::AwesomeRange] +# PREREQ_PM => { } +# VERSION_FROM => q[lib/Seco/AwesomeRange.pm] + +# --- MakeMaker post_initialize section: + + +# --- MakeMaker const_config section: + +# These definitions are from config.sh (via /home/y/lib/perl5/5.8/i686-linux-64int/Config.pm) + +# They may have been overridden via Makefile.PL or on the command line +AR = ar +CC = gcc +CCCDLFLAGS = -fpic +CCDLFLAGS = -Wl,-E -Wl,-R/home/y/lib -Wl,-R/usr/local/lib +DLEXT = so +DLSRC = dl_dlopen.xs +LD = gcc +LDDLFLAGS = -shared -L/usr/local/lib +LDFLAGS = -L/usr/local/lib +LIBC = /lib/libc-2.3.2.so +LIB_EXT = .a +OBJ_EXT = .o +OSNAME = linux +OSVERS = linux +RANLIB = : +SITELIBEXP = /home/y/lib/perl5/site_perl/5.8 +SITEARCHEXP = /home/y/lib/perl5/site_perl/5.8/i686-linux-64int +SO = so +EXE_EXT = +FULL_AR = /usr/bin/ar +VENDORARCHEXP = +VENDORLIBEXP = + + +# --- MakeMaker constants section: +AR_STATIC_ARGS = cr +DIRFILESEP = / +NAME = Seco::AwesomeRange +NAME_SYM = Seco_AwesomeRange +VERSION = 0.01 +VERSION_MACRO = VERSION +VERSION_SYM = 0_01 +DEFINE_VERSION = -D$(VERSION_MACRO)=\"$(VERSION)\" +XS_VERSION = 0.01 +XS_VERSION_MACRO = XS_VERSION +XS_DEFINE_VERSION = -D$(XS_VERSION_MACRO)=\"$(XS_VERSION)\" +INST_ARCHLIB = blib/arch +INST_SCRIPT = blib/script +INST_BIN = blib/bin +INST_LIB = blib/lib +INST_MAN1DIR = blib/man1 +INST_MAN3DIR = blib/man3 +MAN1EXT = 1 +MAN3EXT = 3 +INSTALLDIRS = site +DESTDIR = +PREFIX = +PERLPREFIX = /home/y +SITEPREFIX = /home/y +VENDORPREFIX = +INSTALLPRIVLIB = $(PERLPREFIX)/lib/perl5/5.8 +DESTINSTALLPRIVLIB = $(DESTDIR)$(INSTALLPRIVLIB) +INSTALLSITELIB = $(SITEPREFIX)/lib/perl5/site_perl/5.8 +DESTINSTALLSITELIB = $(DESTDIR)$(INSTALLSITELIB) +INSTALLVENDORLIB = +DESTINSTALLVENDORLIB = $(DESTDIR)$(INSTALLVENDORLIB) +INSTALLARCHLIB = $(PERLPREFIX)/lib/perl5/5.8/i686-linux-64int +DESTINSTALLARCHLIB = $(DESTDIR)$(INSTALLARCHLIB) +INSTALLSITEARCH = $(SITEPREFIX)/lib/perl5/site_perl/5.8/i686-linux-64int +DESTINSTALLSITEARCH = $(DESTDIR)$(INSTALLSITEARCH) +INSTALLVENDORARCH = +DESTINSTALLVENDORARCH = $(DESTDIR)$(INSTALLVENDORARCH) +INSTALLBIN = $(PERLPREFIX)/bin +DESTINSTALLBIN = $(DESTDIR)$(INSTALLBIN) +INSTALLSITEBIN = $(SITEPREFIX)/bin +DESTINSTALLSITEBIN = $(DESTDIR)$(INSTALLSITEBIN) +INSTALLVENDORBIN = +DESTINSTALLVENDORBIN = $(DESTDIR)$(INSTALLVENDORBIN) +INSTALLSCRIPT = $(PERLPREFIX)/bin +DESTINSTALLSCRIPT = $(DESTDIR)$(INSTALLSCRIPT) +INSTALLMAN1DIR = $(PERLPREFIX)/man/man1 +DESTINSTALLMAN1DIR = $(DESTDIR)$(INSTALLMAN1DIR) +INSTALLSITEMAN1DIR = $(SITEPREFIX)/man/man1 +DESTINSTALLSITEMAN1DIR = $(DESTDIR)$(INSTALLSITEMAN1DIR) +INSTALLVENDORMAN1DIR = +DESTINSTALLVENDORMAN1DIR = $(DESTDIR)$(INSTALLVENDORMAN1DIR) +INSTALLMAN3DIR = $(PERLPREFIX)/man/man3 +DESTINSTALLMAN3DIR = $(DESTDIR)$(INSTALLMAN3DIR) +INSTALLSITEMAN3DIR = $(SITEPREFIX)/man/man3 +DESTINSTALLSITEMAN3DIR = $(DESTDIR)$(INSTALLSITEMAN3DIR) +INSTALLVENDORMAN3DIR = +DESTINSTALLVENDORMAN3DIR = $(DESTDIR)$(INSTALLVENDORMAN3DIR) +PERL_LIB = /home/y/lib/perl5/5.8 +PERL_ARCHLIB = /home/y/lib/perl5/5.8/i686-linux-64int +LIBPERL_A = libperl.a +FIRST_MAKEFILE = Makefile +MAKEFILE_OLD = $(FIRST_MAKEFILE).old +MAKE_APERL_FILE = $(FIRST_MAKEFILE).aperl +PERLMAINCC = $(CC) +PERL_INC = /home/y/lib/perl5/5.8/i686-linux-64int/CORE +PERL = /home/y/bin/perl5.8.6 +FULLPERL = /home/y/bin/perl5.8.6 +ABSPERL = $(PERL) +PERLRUN = $(PERL) +FULLPERLRUN = $(FULLPERL) +ABSPERLRUN = $(ABSPERL) +PERLRUNINST = $(PERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)" +FULLPERLRUNINST = $(FULLPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)" +ABSPERLRUNINST = $(ABSPERLRUN) "-I$(INST_ARCHLIB)" "-I$(INST_LIB)" +PERL_CORE = 0 +PERM_RW = 644 +PERM_RWX = 755 + +MAKEMAKER = /home/y/lib/perl5/5.8/ExtUtils/MakeMaker.pm +MM_VERSION = 6.17 +MM_REVISION = 1.133 + +# FULLEXT = Pathname for extension directory (eg Foo/Bar/Oracle). +# BASEEXT = Basename part of FULLEXT. May be just equal FULLEXT. (eg Oracle) +# PARENT_NAME = NAME without BASEEXT and no trailing :: (eg Foo::Bar) +# DLBASE = Basename part of dynamic library. May be just equal BASEEXT. +FULLEXT = Seco/AwesomeRange +BASEEXT = AwesomeRange +PARENT_NAME = Seco +DLBASE = $(BASEEXT) +VERSION_FROM = lib/Seco/AwesomeRange.pm +INC = -I. +DEFINE = +OBJECT = $(BASEEXT)$(OBJ_EXT) +LDFROM = $(OBJECT) +LINKTYPE = dynamic + +# Handy lists of source code files: +XS_FILES = AwesomeRange.xs +C_FILES = AwesomeRange.c +O_FILES = AwesomeRange.o +H_FILES = ppport.h +MAN1PODS = +MAN3PODS = lib/Seco/AwesomeRange.pm + +# Where is the Config information that we are using/depend on +CONFIGDEP = $(PERL_ARCHLIB)$(DIRFILESEP)Config.pm $(PERL_INC)$(DIRFILESEP)config.h + +# Where to build things +INST_LIBDIR = $(INST_LIB)/Seco +INST_ARCHLIBDIR = $(INST_ARCHLIB)/Seco + +INST_AUTODIR = $(INST_LIB)/auto/$(FULLEXT) +INST_ARCHAUTODIR = $(INST_ARCHLIB)/auto/$(FULLEXT) + +INST_STATIC = $(INST_ARCHAUTODIR)/$(BASEEXT)$(LIB_EXT) +INST_DYNAMIC = $(INST_ARCHAUTODIR)/$(DLBASE).$(DLEXT) +INST_BOOT = $(INST_ARCHAUTODIR)/$(BASEEXT).bs + +# Extra linker info +EXPORT_LIST = +PERL_ARCHIVE = +PERL_ARCHIVE_AFTER = + + +TO_INST_PM = lib/Seco/AwesomeRange.pm + +PM_TO_BLIB = lib/Seco/AwesomeRange.pm \ + blib/lib/Seco/AwesomeRange.pm + + +# --- MakeMaker platform_constants section: +MM_Unix_VERSION = 1.42 +PERL_MALLOC_DEF = -DPERL_EXTMALLOC_DEF -Dmalloc=Perl_malloc -Dfree=Perl_mfree -Drealloc=Perl_realloc -Dcalloc=Perl_calloc + + +# --- MakeMaker tool_autosplit section: +# Usage: $(AUTOSPLITFILE) FileToSplit AutoDirToSplitInto +AUTOSPLITFILE = $(PERLRUN) -e 'use AutoSplit; autosplit($$ARGV[0], $$ARGV[1], 0, 1, 1)' + + + +# --- MakeMaker tool_xsubpp section: + +XSUBPPDIR = /home/y/lib/perl5/5.8/ExtUtils +XSUBPP = $(XSUBPPDIR)/xsubpp +XSPROTOARG = +XSUBPPDEPS = /home/y/lib/perl5/5.8/ExtUtils/typemap $(XSUBPP) +XSUBPPARGS = -typemap /home/y/lib/perl5/5.8/ExtUtils/typemap +XSUBPP_EXTRA_ARGS = + + +# --- MakeMaker tools_other section: +SHELL = /bin/sh +CHMOD = chmod +CP = cp +MV = mv +NOOP = $(SHELL) -c true +NOECHO = @ +RM_F = rm -f +RM_RF = rm -rf +TEST_F = test -f +TOUCH = touch +UMASK_NULL = umask 0 +DEV_NULL = > /dev/null 2>&1 +MKPATH = $(PERLRUN) "-MExtUtils::Command" -e mkpath +EQUALIZE_TIMESTAMP = $(PERLRUN) "-MExtUtils::Command" -e eqtime +ECHO = echo +ECHO_N = echo -n +UNINST = 0 +VERBINST = 0 +MOD_INSTALL = $(PERLRUN) -MExtUtils::Install -e 'install({@ARGV}, '\''$(VERBINST)'\'', 0, '\''$(UNINST)'\'');' +DOC_INSTALL = $(PERLRUN) "-MExtUtils::Command::MM" -e perllocal_install +UNINSTALL = $(PERLRUN) "-MExtUtils::Command::MM" -e uninstall +WARN_IF_OLD_PACKLIST = $(PERLRUN) "-MExtUtils::Command::MM" -e warn_if_old_packlist + + +# --- MakeMaker makemakerdflt section: +makemakerdflt: all + $(NOECHO) $(NOOP) + + +# --- MakeMaker dist section: +TAR = tar +TARFLAGS = cvf +ZIP = zip +ZIPFLAGS = -r +COMPRESS = gzip --best +SUFFIX = .gz +SHAR = shar +PREOP = $(NOECHO) $(NOOP) +POSTOP = $(NOECHO) $(NOOP) +TO_UNIX = $(NOECHO) $(NOOP) +CI = ci -u +RCS_LABEL = rcs -Nv$(VERSION_SYM): -q +DIST_CP = best +DIST_DEFAULT = tardist +DISTNAME = Seco-AwesomeRange +DISTVNAME = Seco-AwesomeRange-0.01 + + +# --- MakeMaker macro section: + + +# --- MakeMaker depend section: + + +# --- MakeMaker cflags section: + +CCFLAGS = -fno-strict-aliasing -pipe -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -I/usr/include/gdbm +OPTIMIZE = -O2 +PERLTYPE = +MPOLLUTE = + + +# --- MakeMaker const_loadlibs section: + +# Seco::AwesomeRange might depend on some other libraries: +# See ExtUtils::Liblist for details +# +EXTRALIBS = -lpcre +LDLOADLIBS = -lpcre -lm +BSLOADLIBS = +LD_RUN_PATH = /usr/lib + + +# --- MakeMaker const_cccmd section: +CCCMD = $(CC) -c $(PASTHRU_INC) $(INC) \ + $(CCFLAGS) $(OPTIMIZE) \ + $(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \ + $(XS_DEFINE_VERSION) + +# --- MakeMaker post_constants section: + + +# --- MakeMaker pasthru section: + +PASTHRU = LIB="$(LIB)"\ + LIBPERL_A="$(LIBPERL_A)"\ + LINKTYPE="$(LINKTYPE)"\ + PREFIX="$(PREFIX)"\ + OPTIMIZE="$(OPTIMIZE)"\ + PASTHRU_DEFINE="$(PASTHRU_DEFINE)"\ + PASTHRU_INC="$(PASTHRU_INC)" + + +# --- MakeMaker special_targets section: +.SUFFIXES: .xs .c .C .cpp .i .s .cxx .cc $(OBJ_EXT) + +.PHONY: all config static dynamic test linkext manifest + + + +# --- MakeMaker c_o section: + +.c.i: + gcc -E -c $(PASTHRU_INC) $(INC) \ + $(CCFLAGS) $(OPTIMIZE) \ + $(PERLTYPE) $(MPOLLUTE) $(DEFINE_VERSION) \ + $(XS_DEFINE_VERSION) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c > $*.i + +.c.s: + $(CCCMD) -S $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c + +.c$(OBJ_EXT): + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c + +.C$(OBJ_EXT): + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.C + +.cpp$(OBJ_EXT): + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cpp + +.cxx$(OBJ_EXT): + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cxx + +.cc$(OBJ_EXT): + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.cc + + +# --- MakeMaker xs_c section: + +.xs.c: + $(PERLRUN) $(XSUBPP) $(XSPROTOARG) $(XSUBPPARGS) $(XSUBPP_EXTRA_ARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c + + +# --- MakeMaker xs_o section: + +.xs$(OBJ_EXT): + $(PERLRUN) $(XSUBPP) $(XSPROTOARG) $(XSUBPPARGS) $*.xs > $*.xsc && $(MV) $*.xsc $*.c + $(CCCMD) $(CCCDLFLAGS) "-I$(PERL_INC)" $(PASTHRU_DEFINE) $(DEFINE) $*.c + + +# --- MakeMaker top_targets section: +all :: pure_all manifypods + $(NOECHO) $(NOOP) + + +pure_all :: config pm_to_blib subdirs linkext + $(NOECHO) $(NOOP) + +subdirs :: $(MYEXTLIB) + $(NOECHO) $(NOOP) + +config :: $(FIRST_MAKEFILE) $(INST_LIBDIR)$(DIRFILESEP).exists + $(NOECHO) $(NOOP) + +config :: $(INST_ARCHAUTODIR)$(DIRFILESEP).exists + $(NOECHO) $(NOOP) + +config :: $(INST_AUTODIR)$(DIRFILESEP).exists + $(NOECHO) $(NOOP) + +$(INST_AUTODIR)/.exists :: /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h + $(NOECHO) $(MKPATH) $(INST_AUTODIR) + $(NOECHO) $(EQUALIZE_TIMESTAMP) /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h $(INST_AUTODIR)/.exists + + -$(NOECHO) $(CHMOD) $(PERM_RWX) $(INST_AUTODIR) + +$(INST_LIBDIR)/.exists :: /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h + $(NOECHO) $(MKPATH) $(INST_LIBDIR) + $(NOECHO) $(EQUALIZE_TIMESTAMP) /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h $(INST_LIBDIR)/.exists + + -$(NOECHO) $(CHMOD) $(PERM_RWX) $(INST_LIBDIR) + +$(INST_ARCHAUTODIR)/.exists :: /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h + $(NOECHO) $(MKPATH) $(INST_ARCHAUTODIR) + $(NOECHO) $(EQUALIZE_TIMESTAMP) /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h $(INST_ARCHAUTODIR)/.exists + + -$(NOECHO) $(CHMOD) $(PERM_RWX) $(INST_ARCHAUTODIR) + +config :: $(INST_MAN3DIR)$(DIRFILESEP).exists + $(NOECHO) $(NOOP) + + +$(INST_MAN3DIR)/.exists :: /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h + $(NOECHO) $(MKPATH) $(INST_MAN3DIR) + $(NOECHO) $(EQUALIZE_TIMESTAMP) /home/y/lib/perl5/5.8/i686-linux-64int/CORE/perl.h $(INST_MAN3DIR)/.exists + + -$(NOECHO) $(CHMOD) $(PERM_RWX) $(INST_MAN3DIR) + +$(O_FILES): $(H_FILES) + +help: + perldoc ExtUtils::MakeMaker + + +# --- MakeMaker linkext section: + +linkext :: $(LINKTYPE) + $(NOECHO) $(NOOP) + + +# --- MakeMaker dlsyms section: + + +# --- MakeMaker dynamic section: + +dynamic :: $(FIRST_MAKEFILE) $(INST_DYNAMIC) $(INST_BOOT) + $(NOECHO) $(NOOP) + + +# --- MakeMaker dynamic_bs section: +BOOTSTRAP = $(BASEEXT).bs + +# As Mkbootstrap might not write a file (if none is required) +# we use touch to prevent make continually trying to remake it. +# The DynaLoader only reads a non-empty file. +$(BOOTSTRAP): $(FIRST_MAKEFILE) $(BOOTDEP) $(INST_ARCHAUTODIR)$(DIRFILESEP).exists + $(NOECHO) $(ECHO) "Running Mkbootstrap for $(NAME) ($(BSLOADLIBS))" + $(NOECHO) $(PERLRUN) \ + "-MExtUtils::Mkbootstrap" \ + -e "Mkbootstrap('$(BASEEXT)','$(BSLOADLIBS)');" + $(NOECHO) $(TOUCH) $(BOOTSTRAP) + $(CHMOD) $(PERM_RW) $@ + +$(INST_BOOT): $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DIRFILESEP).exists + $(NOECHO) $(RM_RF) $(INST_BOOT) + -$(CP) $(BOOTSTRAP) $(INST_BOOT) + $(CHMOD) $(PERM_RW) $@ + + +# --- MakeMaker dynamic_lib section: + +# This section creates the dynamically loadable $(INST_DYNAMIC) +# from $(OBJECT) and possibly $(MYEXTLIB). +ARMAYBE = : +OTHERLDFLAGS = +INST_DYNAMIC_DEP = +INST_DYNAMIC_FIX = + +$(INST_DYNAMIC): $(OBJECT) $(MYEXTLIB) $(BOOTSTRAP) $(INST_ARCHAUTODIR)$(DIRFILESEP).exists $(EXPORT_LIST) $(PERL_ARCHIVE) $(PERL_ARCHIVE_AFTER) $(INST_DYNAMIC_DEP) + $(RM_F) $@ + LD_RUN_PATH="$(LD_RUN_PATH)" $(LD) $(LDDLFLAGS) $(LDFROM) $(OTHERLDFLAGS) -o $@ $(MYEXTLIB) $(PERL_ARCHIVE) $(LDLOADLIBS) $(PERL_ARCHIVE_AFTER) $(EXPORT_LIST) $(INST_DYNAMIC_FIX) + $(CHMOD) $(PERM_RWX) $@ + + +# --- MakeMaker static section: + +## $(INST_PM) has been moved to the all: target. +## It remains here for awhile to allow for old usage: "make static" +static :: $(FIRST_MAKEFILE) $(INST_STATIC) + $(NOECHO) $(NOOP) + + +# --- MakeMaker static_lib section: + +$(INST_STATIC): $(OBJECT) $(MYEXTLIB) $(INST_ARCHAUTODIR)$(DIRFILESEP).exists + $(RM_RF) $@ + $(FULL_AR) $(AR_STATIC_ARGS) $@ $(OBJECT) && $(RANLIB) $@ + $(CHMOD) $(PERM_RWX) $@ + $(NOECHO) $(ECHO) "$(EXTRALIBS)" > $(INST_ARCHAUTODIR)/extralibs.ld + + + +# --- MakeMaker manifypods section: + +POD2MAN_EXE = $(PERLRUN) "-MExtUtils::Command::MM" -e pod2man "--" +POD2MAN = $(POD2MAN_EXE) + + +manifypods : pure_all \ + lib/Seco/AwesomeRange.pm \ + lib/Seco/AwesomeRange.pm + $(NOECHO) $(POD2MAN) --section=3 --perm_rw=$(PERM_RW)\ + lib/Seco/AwesomeRange.pm $(INST_MAN3DIR)/Seco::AwesomeRange.$(MAN3EXT) + + + + +# --- MakeMaker processPL section: + + +# --- MakeMaker installbin section: + + +# --- MakeMaker subdirs section: + +# none + +# --- MakeMaker clean_subdirs section: +clean_subdirs : + $(NOECHO) $(NOOP) + + +# --- MakeMaker clean section: + +# Delete temporary files but do not touch installed files. We don't delete +# the Makefile here so a later make realclean still has a makefile to use. + +clean :: clean_subdirs + -$(RM_RF) AwesomeRange.c ./blib $(MAKE_APERL_FILE) $(INST_ARCHAUTODIR)/extralibs.all $(INST_ARCHAUTODIR)/extralibs.ld perlmain.c tmon.out mon.out so_locations pm_to_blib *$(OBJ_EXT) *$(LIB_EXT) perl.exe perl perl$(EXE_EXT) $(BOOTSTRAP) $(BASEEXT).bso $(BASEEXT).def lib$(BASEEXT).def $(BASEEXT).exp $(BASEEXT).x core core.*perl.*.? *perl.core core.[0-9] core.[0-9][0-9] core.[0-9][0-9][0-9] core.[0-9][0-9][0-9][0-9] core.[0-9][0-9][0-9][0-9][0-9] + -$(MV) $(FIRST_MAKEFILE) $(MAKEFILE_OLD) $(DEV_NULL) + + +# --- MakeMaker realclean_subdirs section: +realclean_subdirs : + $(NOECHO) $(NOOP) + + +# --- MakeMaker realclean section: + +# Delete temporary files (via clean) and also delete installed files +realclean purge :: clean realclean_subdirs + $(RM_RF) $(INST_AUTODIR) $(INST_ARCHAUTODIR) + $(RM_RF) $(DISTVNAME) + $(RM_F) $(INST_DYNAMIC) $(INST_BOOT) + $(RM_F) $(INST_STATIC) + $(RM_F) blib/lib/Seco/AwesomeRange.pm $(MAKEFILE_OLD) $(FIRST_MAKEFILE) + + +# --- MakeMaker metafile section: +metafile : + $(NOECHO) $(ECHO) '# http://module-build.sourceforge.net/META-spec.html' > META.yml + $(NOECHO) $(ECHO) '#XXXXXXX This is a prototype!!! It will change in the future!!! XXXXX#' >> META.yml + $(NOECHO) $(ECHO) 'name: Seco-AwesomeRange' >> META.yml + $(NOECHO) $(ECHO) 'version: 0.01' >> META.yml + $(NOECHO) $(ECHO) 'version_from: lib/Seco/AwesomeRange.pm' >> META.yml + $(NOECHO) $(ECHO) 'installdirs: site' >> META.yml + $(NOECHO) $(ECHO) 'requires:' >> META.yml + $(NOECHO) $(ECHO) '' >> META.yml + $(NOECHO) $(ECHO) 'distribution_type: module' >> META.yml + $(NOECHO) $(ECHO) 'generated_by: ExtUtils::MakeMaker version 6.17' >> META.yml + + +# --- MakeMaker metafile_addtomanifest section: +metafile_addtomanifest: + $(NOECHO) $(PERLRUN) -MExtUtils::Manifest=maniadd -e 'eval { maniadd({q{META.yml} => q{Module meta-data (added by MakeMaker)}}) } ' \ + -e ' or print "Could not add META.yml to MANIFEST: $${'\''@'\''}\n"' + + +# --- MakeMaker dist_basics section: +distclean :: realclean distcheck + $(NOECHO) $(NOOP) + +distcheck : + $(PERLRUN) "-MExtUtils::Manifest=fullcheck" -e fullcheck + +skipcheck : + $(PERLRUN) "-MExtUtils::Manifest=skipcheck" -e skipcheck + +manifest : + $(PERLRUN) "-MExtUtils::Manifest=mkmanifest" -e mkmanifest + +veryclean : realclean + $(RM_F) *~ *.orig */*~ */*.orig + + + +# --- MakeMaker dist_core section: + +dist : $(DIST_DEFAULT) $(FIRST_MAKEFILE) + $(NOECHO) $(PERLRUN) -l -e 'print '\''Warning: Makefile possibly out of date with $(VERSION_FROM)'\''' \ + -e ' if -e '\''$(VERSION_FROM)'\'' and -M '\''$(VERSION_FROM)'\'' < -M '\''$(FIRST_MAKEFILE)'\'';' + +tardist : $(DISTVNAME).tar$(SUFFIX) + $(NOECHO) $(NOOP) + +uutardist : $(DISTVNAME).tar$(SUFFIX) + uuencode $(DISTVNAME).tar$(SUFFIX) $(DISTVNAME).tar$(SUFFIX) > $(DISTVNAME).tar$(SUFFIX)_uu + +$(DISTVNAME).tar$(SUFFIX) : distdir + $(PREOP) + $(TO_UNIX) + $(TAR) $(TARFLAGS) $(DISTVNAME).tar $(DISTVNAME) + $(RM_RF) $(DISTVNAME) + $(COMPRESS) $(DISTVNAME).tar + $(POSTOP) + +zipdist : $(DISTVNAME).zip + $(NOECHO) $(NOOP) + +$(DISTVNAME).zip : distdir + $(PREOP) + $(ZIP) $(ZIPFLAGS) $(DISTVNAME).zip $(DISTVNAME) + $(RM_RF) $(DISTVNAME) + $(POSTOP) + +shdist : distdir + $(PREOP) + $(SHAR) $(DISTVNAME) > $(DISTVNAME).shar + $(RM_RF) $(DISTVNAME) + $(POSTOP) + + +# --- MakeMaker distdir section: +distdir : metafile metafile_addtomanifest + $(RM_RF) $(DISTVNAME) + $(PERLRUN) "-MExtUtils::Manifest=manicopy,maniread" \ + -e "manicopy(maniread(),'$(DISTVNAME)', '$(DIST_CP)');" + + + +# --- MakeMaker dist_test section: + +disttest : distdir + cd $(DISTVNAME) && $(ABSPERLRUN) Makefile.PL + cd $(DISTVNAME) && $(MAKE) $(PASTHRU) + cd $(DISTVNAME) && $(MAKE) test $(PASTHRU) + + +# --- MakeMaker dist_ci section: + +ci : + $(PERLRUN) "-MExtUtils::Manifest=maniread" \ + -e "@all = keys %{ maniread() };" \ + -e "print(qq{Executing $(CI) @all\n}); system(qq{$(CI) @all});" \ + -e "print(qq{Executing $(RCS_LABEL) ...\n}); system(qq{$(RCS_LABEL) @all});" + + +# --- MakeMaker install section: + +install :: all pure_install doc_install + +install_perl :: all pure_perl_install doc_perl_install + +install_site :: all pure_site_install doc_site_install + +install_vendor :: all pure_vendor_install doc_vendor_install + +pure_install :: pure_$(INSTALLDIRS)_install + +doc_install :: doc_$(INSTALLDIRS)_install + +pure__install : pure_site_install + $(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site + +doc__install : doc_site_install + $(NOECHO) $(ECHO) INSTALLDIRS not defined, defaulting to INSTALLDIRS=site + +pure_perl_install :: + $(NOECHO) $(MOD_INSTALL) \ + read $(PERL_ARCHLIB)/auto/$(FULLEXT)/.packlist \ + write $(DESTINSTALLARCHLIB)/auto/$(FULLEXT)/.packlist \ + $(INST_LIB) $(DESTINSTALLPRIVLIB) \ + $(INST_ARCHLIB) $(DESTINSTALLARCHLIB) \ + $(INST_BIN) $(DESTINSTALLBIN) \ + $(INST_SCRIPT) $(DESTINSTALLSCRIPT) \ + $(INST_MAN1DIR) $(DESTINSTALLMAN1DIR) \ + $(INST_MAN3DIR) $(DESTINSTALLMAN3DIR) + $(NOECHO) $(WARN_IF_OLD_PACKLIST) \ + $(SITEARCHEXP)/auto/$(FULLEXT) + + +pure_site_install :: + $(NOECHO) $(MOD_INSTALL) \ + read $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist \ + write $(DESTINSTALLSITEARCH)/auto/$(FULLEXT)/.packlist \ + $(INST_LIB) $(DESTINSTALLSITELIB) \ + $(INST_ARCHLIB) $(DESTINSTALLSITEARCH) \ + $(INST_BIN) $(DESTINSTALLSITEBIN) \ + $(INST_SCRIPT) $(DESTINSTALLSCRIPT) \ + $(INST_MAN1DIR) $(DESTINSTALLSITEMAN1DIR) \ + $(INST_MAN3DIR) $(DESTINSTALLSITEMAN3DIR) + $(NOECHO) $(WARN_IF_OLD_PACKLIST) \ + $(PERL_ARCHLIB)/auto/$(FULLEXT) + +pure_vendor_install :: + $(NOECHO) $(MOD_INSTALL) \ + read $(VENDORARCHEXP)/auto/$(FULLEXT)/.packlist \ + write $(DESTINSTALLVENDORARCH)/auto/$(FULLEXT)/.packlist \ + $(INST_LIB) $(DESTINSTALLVENDORLIB) \ + $(INST_ARCHLIB) $(DESTINSTALLVENDORARCH) \ + $(INST_BIN) $(DESTINSTALLVENDORBIN) \ + $(INST_SCRIPT) $(DESTINSTALLSCRIPT) \ + $(INST_MAN1DIR) $(DESTINSTALLVENDORMAN1DIR) \ + $(INST_MAN3DIR) $(DESTINSTALLVENDORMAN3DIR) + +doc_perl_install :: + $(NOECHO) $(ECHO) Appending installation info to $(DESTINSTALLARCHLIB)/perllocal.pod + -$(NOECHO) $(MKPATH) $(DESTINSTALLARCHLIB) + -$(NOECHO) $(DOC_INSTALL) \ + "Module" "$(NAME)" \ + "installed into" "$(INSTALLPRIVLIB)" \ + LINKTYPE "$(LINKTYPE)" \ + VERSION "$(VERSION)" \ + EXE_FILES "$(EXE_FILES)" \ + >> $(DESTINSTALLARCHLIB)/perllocal.pod + +doc_site_install :: + $(NOECHO) $(ECHO) Appending installation info to $(DESTINSTALLARCHLIB)/perllocal.pod + -$(NOECHO) $(MKPATH) $(DESTINSTALLARCHLIB) + -$(NOECHO) $(DOC_INSTALL) \ + "Module" "$(NAME)" \ + "installed into" "$(INSTALLSITELIB)" \ + LINKTYPE "$(LINKTYPE)" \ + VERSION "$(VERSION)" \ + EXE_FILES "$(EXE_FILES)" \ + >> $(DESTINSTALLARCHLIB)/perllocal.pod + +doc_vendor_install :: + $(NOECHO) $(ECHO) Appending installation info to $(DESTINSTALLARCHLIB)/perllocal.pod + -$(NOECHO) $(MKPATH) $(DESTINSTALLARCHLIB) + -$(NOECHO) $(DOC_INSTALL) \ + "Module" "$(NAME)" \ + "installed into" "$(INSTALLVENDORLIB)" \ + LINKTYPE "$(LINKTYPE)" \ + VERSION "$(VERSION)" \ + EXE_FILES "$(EXE_FILES)" \ + >> $(DESTINSTALLARCHLIB)/perllocal.pod + + +uninstall :: uninstall_from_$(INSTALLDIRS)dirs + +uninstall_from_perldirs :: + $(NOECHO) $(UNINSTALL) $(PERL_ARCHLIB)/auto/$(FULLEXT)/.packlist + +uninstall_from_sitedirs :: + $(NOECHO) $(UNINSTALL) $(SITEARCHEXP)/auto/$(FULLEXT)/.packlist + +uninstall_from_vendordirs :: + $(NOECHO) $(UNINSTALL) $(VENDORARCHEXP)/auto/$(FULLEXT)/.packlist + + +# --- MakeMaker force section: +# Phony target to force checking subdirectories. +FORCE: + $(NOECHO) $(NOOP) + + +# --- MakeMaker perldepend section: + +PERL_HDRS = \ + $(PERL_INC)/EXTERN.h \ + $(PERL_INC)/INTERN.h \ + $(PERL_INC)/XSUB.h \ + $(PERL_INC)/av.h \ + $(PERL_INC)/cc_runtime.h \ + $(PERL_INC)/config.h \ + $(PERL_INC)/cop.h \ + $(PERL_INC)/cv.h \ + $(PERL_INC)/dosish.h \ + $(PERL_INC)/embed.h \ + $(PERL_INC)/embedvar.h \ + $(PERL_INC)/fakethr.h \ + $(PERL_INC)/form.h \ + $(PERL_INC)/gv.h \ + $(PERL_INC)/handy.h \ + $(PERL_INC)/hv.h \ + $(PERL_INC)/intrpvar.h \ + $(PERL_INC)/iperlsys.h \ + $(PERL_INC)/keywords.h \ + $(PERL_INC)/mg.h \ + $(PERL_INC)/nostdio.h \ + $(PERL_INC)/op.h \ + $(PERL_INC)/opcode.h \ + $(PERL_INC)/patchlevel.h \ + $(PERL_INC)/perl.h \ + $(PERL_INC)/perlio.h \ + $(PERL_INC)/perlsdio.h \ + $(PERL_INC)/perlsfio.h \ + $(PERL_INC)/perlvars.h \ + $(PERL_INC)/perly.h \ + $(PERL_INC)/pp.h \ + $(PERL_INC)/pp_proto.h \ + $(PERL_INC)/proto.h \ + $(PERL_INC)/regcomp.h \ + $(PERL_INC)/regexp.h \ + $(PERL_INC)/regnodes.h \ + $(PERL_INC)/scope.h \ + $(PERL_INC)/sv.h \ + $(PERL_INC)/thrdvar.h \ + $(PERL_INC)/thread.h \ + $(PERL_INC)/unixish.h \ + $(PERL_INC)/util.h + +$(OBJECT) : $(PERL_HDRS) + +AwesomeRange.c : $(XSUBPPDEPS) + + +# --- MakeMaker makefile section: + +$(OBJECT) : $(FIRST_MAKEFILE) + +# We take a very conservative approach here, but it's worth it. +# We move Makefile to Makefile.old here to avoid gnu make looping. +$(FIRST_MAKEFILE) : Makefile.PL $(CONFIGDEP) + $(NOECHO) $(ECHO) "Makefile out-of-date with respect to $?" + $(NOECHO) $(ECHO) "Cleaning current config before rebuilding Makefile..." + $(NOECHO) $(RM_F) $(MAKEFILE_OLD) + $(NOECHO) $(MV) $(FIRST_MAKEFILE) $(MAKEFILE_OLD) + -$(MAKE) -f $(MAKEFILE_OLD) clean $(DEV_NULL) || $(NOOP) + $(PERLRUN) Makefile.PL + $(NOECHO) $(ECHO) "==> Your Makefile has been rebuilt. <==" + $(NOECHO) $(ECHO) "==> Please rerun the make command. <==" + false + + + +# --- MakeMaker staticmake section: + +# --- MakeMaker makeaperl section --- +MAP_TARGET = perl +FULLPERL = /home/y/bin/perl5.8.6 + +$(MAP_TARGET) :: static $(MAKE_APERL_FILE) + $(MAKE) -f $(MAKE_APERL_FILE) $@ + +$(MAKE_APERL_FILE) : $(FIRST_MAKEFILE) + $(NOECHO) $(ECHO) Writing \"$(MAKE_APERL_FILE)\" for this $(MAP_TARGET) + $(NOECHO) $(PERLRUNINST) \ + Makefile.PL DIR= \ + MAKEFILE=$(MAKE_APERL_FILE) LINKTYPE=static \ + MAKEAPERL=1 NORECURS=1 CCCDLFLAGS= + + +# --- MakeMaker test section: + +TEST_VERBOSE=0 +TEST_TYPE=test_$(LINKTYPE) +TEST_FILE = test.pl +TEST_FILES = t/*.t +TESTDB_SW = -d + +testdb :: testdb_$(LINKTYPE) + +test :: $(TEST_TYPE) + +test_dynamic :: pure_all + PERL_DL_NONLAZY=1 $(FULLPERLRUN) "-MExtUtils::Command::MM" "-e" "test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')" $(TEST_FILES) + +testdb_dynamic :: pure_all + PERL_DL_NONLAZY=1 $(FULLPERLRUN) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE) + +test_ : test_dynamic + +test_static :: pure_all $(MAP_TARGET) + PERL_DL_NONLAZY=1 ./$(MAP_TARGET) "-MExtUtils::Command::MM" "-e" "test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')" $(TEST_FILES) + +testdb_static :: pure_all $(MAP_TARGET) + PERL_DL_NONLAZY=1 ./$(MAP_TARGET) $(TESTDB_SW) "-I$(INST_LIB)" "-I$(INST_ARCHLIB)" $(TEST_FILE) + + + +# --- MakeMaker ppd section: +# Creates a PPD (Perl Package Description) for a binary distribution. +ppd: + $(NOECHO) $(ECHO) '' > $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' $(DISTNAME)' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' Perl extension for dealing with Seco ranges' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' Evan Miller <eam@inktomisearch.com>' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' ' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' ' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' ' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' ' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) ' ' >> $(DISTNAME).ppd + $(NOECHO) $(ECHO) '' >> $(DISTNAME).ppd + + +# --- MakeMaker pm_to_blib section: + +pm_to_blib: $(TO_INST_PM) + $(NOECHO) $(PERLRUN) -MExtUtils::Install -e 'pm_to_blib({@ARGV}, '\''$(INST_LIB)/auto'\'', '\''$(PM_FILTER)'\'')'\ + lib/Seco/AwesomeRange.pm blib/lib/Seco/AwesomeRange.pm + $(NOECHO) $(TOUCH) $@ + +# --- MakeMaker selfdocument section: + + +# --- MakeMaker postamble section: + + +# End. diff --git a/seco_awesomerange/source/Makefile.PL b/seco_awesomerange/source/Makefile.PL new file mode 100644 index 0000000..0a8ff3b --- /dev/null +++ b/seco_awesomerange/source/Makefile.PL @@ -0,0 +1,40 @@ +use 5.008004; +use ExtUtils::MakeMaker; +# See lib/ExtUtils/MakeMaker.pm for details of how to influence +# the contents of the Makefile that is written. +WriteMakefile( + NAME => 'Seco::AwesomeRange', + VERSION_FROM => 'lib/Seco/AwesomeRange.pm', # finds $VERSION + PREREQ_PM => {}, # e.g., Module::Name => 1.1 + ($] >= 5.005 ? ## Add these new keywords supported since 5.005 + (ABSTRACT_FROM => 'lib/Seco/AwesomeRange.pm', # retrieve abstract from module + AUTHOR => 'Evan Miller ') : ()), + LIBS => ['-lrange -lpcre -lm'], # e.g., '-lm' + DEFINE => '', # e.g., '-DHAVE_SOMETHING' + INC => '-I.', # e.g., '-I. -I/usr/include/other' + # Un-comment this if you add C files to link with later: + # OBJECT => '$(O_FILES)', # link all the C files too +); +if (eval {require ExtUtils::Constant; 1}) { + # If you edit these definitions to change the constants used by this module, + # you will need to use the generated const-c.inc and const-xs.inc + # files to replace their "fallback" counterparts before distributing your + # changes. + my @names = (qw()); + ExtUtils::Constant::WriteConstants( + NAME => 'Seco::AwesomeRange', + NAMES => \@names, + DEFAULT_TYPE => 'IV', + C_FILE => 'const-c.inc', + XS_FILE => 'const-xs.inc', + ); + +} +else { + use File::Copy; + use File::Spec; + foreach my $file ('const-c.inc', 'const-xs.inc') { + my $fallback = File::Spec->catfile('fallback', $file); + copy ($fallback, $file) or die "Can't copy $fallback to $file: $!"; + } +} diff --git a/seco_awesomerange/source/README b/seco_awesomerange/source/README new file mode 100644 index 0000000..3007eac --- /dev/null +++ b/seco_awesomerange/source/README @@ -0,0 +1,38 @@ +Seco-AwesomeRange version 0.01 +============================== + +The README is used to introduce the module and provide instructions on +how to install the module, any machine dependencies it may have (for +example C compilers and installed libraries) and any other information +that should be provided before the module is installed. + +A README file is required for CPAN modules since CPAN extracts the +README file from a module distribution so that people browsing the +archive can use it get an idea of the modules uses. It is usually a +good idea to provide version information here so that people can +decide whether fixes for the module are worth downloading. + +INSTALLATION + +To install this module type the following: + + perl Makefile.PL + make + make test + make install + +DEPENDENCIES + +This module requires these other modules and libraries: + + blah blah blah + +COPYRIGHT AND LICENCE + +Put the correct copyright and licence information here. + +Copyright (C) 2005 by Yahoo! Inc. + +Copyright (c) 2011, Yahoo! Inc. All rights reserved. +Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + diff --git a/seco_awesomerange/source/const-c.inc b/seco_awesomerange/source/const-c.inc new file mode 100644 index 0000000..47fd0ec --- /dev/null +++ b/seco_awesomerange/source/const-c.inc @@ -0,0 +1,55 @@ +#define PERL_constant_NOTFOUND 1 +#define PERL_constant_NOTDEF 2 +#define PERL_constant_ISIV 3 +#define PERL_constant_ISNO 4 +#define PERL_constant_ISNV 5 +#define PERL_constant_ISPV 6 +#define PERL_constant_ISPVN 7 +#define PERL_constant_ISSV 8 +#define PERL_constant_ISUNDEF 9 +#define PERL_constant_ISUV 10 +#define PERL_constant_ISYES 11 + +#ifndef NVTYPE +typedef double NV; /* 5.6 and later define NVTYPE, and typedef NV to it. */ +#endif +#ifndef aTHX_ +#define aTHX_ /* 5.6 or later define this for threading support. */ +#endif +#ifndef pTHX_ +#define pTHX_ /* 5.6 or later define this for threading support. */ +#endif + +static int +constant (pTHX_ const char *name, STRLEN len) { + /* Initially switch on the length of the name. */ + /* When generated this function returned values for the list of names given + in this section of perl code. Rather than manually editing these functions + to add or remove constants, which would result in this comment and section + of code becoming inaccurate, we recommend that you edit this section of + code, and use it to regenerate a new set of constant functions which you + then use to replace the originals. + + Regenerate these constant functions by feeding this entire source file to + perl -x + +#!/home/y/bin/perl5.8.6 -w +use ExtUtils::Constant qw (constant_types C_constant XS_constant); + +my $types = {map {($_, 1)} qw()}; +my @names = (qw()); + +print constant_types(); # macro defs +foreach (C_constant ("Seco::AwesomeRange", 'constant', 'IV', $types, undef, 3, @names) ) { + print $_, "\n"; # C constant subs +} +print "#### XS Section:\n"; +print XS_constant ("Seco::AwesomeRange", $types); +__END__ + */ + + switch (len) { + } + return PERL_constant_NOTFOUND; +} + diff --git a/seco_awesomerange/source/const-xs.inc b/seco_awesomerange/source/const-xs.inc new file mode 100644 index 0000000..74361c5 --- /dev/null +++ b/seco_awesomerange/source/const-xs.inc @@ -0,0 +1,87 @@ +void +constant(sv) + PREINIT: +#ifdef dXSTARG + dXSTARG; /* Faster if we have it. */ +#else + dTARGET; +#endif + STRLEN len; + int type; + /* IV iv; Uncomment this if you need to return IVs */ + /* NV nv; Uncomment this if you need to return NVs */ + /* const char *pv; Uncomment this if you need to return PVs */ + INPUT: + SV * sv; + const char * s = SvPV(sv, len); + PPCODE: + type = constant(aTHX_ s, len); + /* Return 1 or 2 items. First is error message, or undef if no error. + Second, if present, is found value */ + switch (type) { + case PERL_constant_NOTFOUND: + sv = sv_2mortal(newSVpvf("%s is not a valid Seco::AwesomeRange macro", s)); + PUSHs(sv); + break; + case PERL_constant_NOTDEF: + sv = sv_2mortal(newSVpvf( + "Your vendor has not defined Seco::AwesomeRange macro %s, used", s)); + PUSHs(sv); + break; + /* Uncomment this if you need to return IVs + case PERL_constant_ISIV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHi(iv); + break; */ + /* Uncomment this if you need to return NOs + case PERL_constant_ISNO: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(&PL_sv_no); + break; */ + /* Uncomment this if you need to return NVs + case PERL_constant_ISNV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHn(nv); + break; */ + /* Uncomment this if you need to return PVs + case PERL_constant_ISPV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHp(pv, strlen(pv)); + break; */ + /* Uncomment this if you need to return PVNs + case PERL_constant_ISPVN: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHp(pv, iv); + break; */ + /* Uncomment this if you need to return SVs + case PERL_constant_ISSV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(sv); + break; */ + /* Uncomment this if you need to return UNDEFs + case PERL_constant_ISUNDEF: + break; */ + /* Uncomment this if you need to return UVs + case PERL_constant_ISUV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHu((UV)iv); + break; */ + /* Uncomment this if you need to return YESs + case PERL_constant_ISYES: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(&PL_sv_yes); + break; */ + default: + sv = sv_2mortal(newSVpvf( + "Unexpected return type %d while processing Seco::AwesomeRange macro %s, used", + type, s)); + PUSHs(sv); + } diff --git a/seco_awesomerange/source/fallback/const-c.inc b/seco_awesomerange/source/fallback/const-c.inc new file mode 100644 index 0000000..12b67a1 --- /dev/null +++ b/seco_awesomerange/source/fallback/const-c.inc @@ -0,0 +1,55 @@ +#define PERL_constant_NOTFOUND 1 +#define PERL_constant_NOTDEF 2 +#define PERL_constant_ISIV 3 +#define PERL_constant_ISNO 4 +#define PERL_constant_ISNV 5 +#define PERL_constant_ISPV 6 +#define PERL_constant_ISPVN 7 +#define PERL_constant_ISSV 8 +#define PERL_constant_ISUNDEF 9 +#define PERL_constant_ISUV 10 +#define PERL_constant_ISYES 11 + +#ifndef NVTYPE +typedef double NV; /* 5.6 and later define NVTYPE, and typedef NV to it. */ +#endif +#ifndef aTHX_ +#define aTHX_ /* 5.6 or later define this for threading support. */ +#endif +#ifndef pTHX_ +#define pTHX_ /* 5.6 or later define this for threading support. */ +#endif + +static int +constant (pTHX_ const char *name, STRLEN len) { + /* Initially switch on the length of the name. */ + /* When generated this function returned values for the list of names given + in this section of perl code. Rather than manually editing these functions + to add or remove constants, which would result in this comment and section + of code becoming inaccurate, we recommend that you edit this section of + code, and use it to regenerate a new set of constant functions which you + then use to replace the originals. + + Regenerate these constant functions by feeding this entire source file to + perl -x + +#!/usr/local/bin/perl -w +use ExtUtils::Constant qw (constant_types C_constant XS_constant); + +my $types = {map {($_, 1)} qw()}; +my @names = (qw()); + +print constant_types(); # macro defs +foreach (C_constant ("Seco::AwesomeRange", 'constant', 'IV', $types, undef, 3, @names) ) { + print $_, "\n"; # C constant subs +} +print "#### XS Section:\n"; +print XS_constant ("Seco::AwesomeRange", $types); +__END__ + */ + + switch (len) { + } + return PERL_constant_NOTFOUND; +} + diff --git a/seco_awesomerange/source/fallback/const-xs.inc b/seco_awesomerange/source/fallback/const-xs.inc new file mode 100644 index 0000000..74361c5 --- /dev/null +++ b/seco_awesomerange/source/fallback/const-xs.inc @@ -0,0 +1,87 @@ +void +constant(sv) + PREINIT: +#ifdef dXSTARG + dXSTARG; /* Faster if we have it. */ +#else + dTARGET; +#endif + STRLEN len; + int type; + /* IV iv; Uncomment this if you need to return IVs */ + /* NV nv; Uncomment this if you need to return NVs */ + /* const char *pv; Uncomment this if you need to return PVs */ + INPUT: + SV * sv; + const char * s = SvPV(sv, len); + PPCODE: + type = constant(aTHX_ s, len); + /* Return 1 or 2 items. First is error message, or undef if no error. + Second, if present, is found value */ + switch (type) { + case PERL_constant_NOTFOUND: + sv = sv_2mortal(newSVpvf("%s is not a valid Seco::AwesomeRange macro", s)); + PUSHs(sv); + break; + case PERL_constant_NOTDEF: + sv = sv_2mortal(newSVpvf( + "Your vendor has not defined Seco::AwesomeRange macro %s, used", s)); + PUSHs(sv); + break; + /* Uncomment this if you need to return IVs + case PERL_constant_ISIV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHi(iv); + break; */ + /* Uncomment this if you need to return NOs + case PERL_constant_ISNO: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(&PL_sv_no); + break; */ + /* Uncomment this if you need to return NVs + case PERL_constant_ISNV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHn(nv); + break; */ + /* Uncomment this if you need to return PVs + case PERL_constant_ISPV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHp(pv, strlen(pv)); + break; */ + /* Uncomment this if you need to return PVNs + case PERL_constant_ISPVN: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHp(pv, iv); + break; */ + /* Uncomment this if you need to return SVs + case PERL_constant_ISSV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(sv); + break; */ + /* Uncomment this if you need to return UNDEFs + case PERL_constant_ISUNDEF: + break; */ + /* Uncomment this if you need to return UVs + case PERL_constant_ISUV: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHu((UV)iv); + break; */ + /* Uncomment this if you need to return YESs + case PERL_constant_ISYES: + EXTEND(SP, 1); + PUSHs(&PL_sv_undef); + PUSHs(&PL_sv_yes); + break; */ + default: + sv = sv_2mortal(newSVpvf( + "Unexpected return type %d while processing Seco::AwesomeRange macro %s, used", + type, s)); + PUSHs(sv); + } diff --git a/seco_awesomerange/source/lib/Seco/AwesomeRange.pm b/seco_awesomerange/source/lib/Seco/AwesomeRange.pm new file mode 100644 index 0000000..eda81b2 --- /dev/null +++ b/seco_awesomerange/source/lib/Seco/AwesomeRange.pm @@ -0,0 +1,335 @@ +package Seco::AwesomeRange; + +# Copyright (c) 2011, Yahoo! Inc. All rights reserved. +# Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. + +use 5.005_03; +use strict; +use warnings; +use Carp; + +require Exporter; +use AutoLoader; + +our @ISA = qw(Exporter); + +our @EXPORT_OK = qw/ expand_range range_set_altpath + compress_range sorted_expand_range + nodes_parser want_warnings want_caching + clear_caches get_version + expand compress sorted_expand test_range + /; + +our %EXPORT_TAGS = ( + 'all' => [ @EXPORT_OK ], + 'common' => [ qw/ expand_range compress_range + sorted_expand_range nodes_parser test_range / ], + ); + +our @EXPORT = qw( ); + +our $VERSION = '0.01'; + +sub AUTOLOAD { + # This AUTOLOAD is used to 'autoload' constants from the constant() + # XS function. + + my $constname; + our $AUTOLOAD; + ($constname = $AUTOLOAD) =~ s/.*:://; + croak "&Seco::AwesomeRange::constant not defined" if $constname eq 'constant'; + my ($error, $val) = constant($constname); + if ($error) { croak $error; } + { + no strict 'refs'; + *$AUTOLOAD = sub { $val }; + } + goto &$AUTOLOAD; +} + +our $raise_exceptions = 0; +our $errno = 0; +require XSLoader; +XSLoader::load('Seco::AwesomeRange', $VERSION); + +# Preloaded methods go here. + +*expand_range = \&range_expand; +*expand = \&range_expand; +*sorted_expand_range = \&range_expand_sorted; +*sorted_expand = \&range_expand_sorted; +*want_caching = \&range_want_caching; +*want_warnings_real = \&range_want_warnings; +*get_exception = \&range_get_exception; +*clear_exception = \&range_clear_exception; +*clear_caches = \&range_clear_caches; +*get_version = \&range_get_version; + + +sub range_compress { + my $ref = ref $_[0]; + + if (not $ref) { + return range_compress_xs(\@_, ","); + } elsif ($ref eq "ARRAY") { + return range_compress_xs($_[0], ",") + } elsif ($ref ne "HASH") { + croak "range_compress can only be called with an array, ref to array, or ref to hash"; + } + + # hash ref + my %settings = ( + separator => ',', + level => 1, + readable => 0, + %{$_[0]} + ); + + my $nodes = $settings{nodes} || $settings{hosts}; + croak "range_compress: the 'nodes' argument is required." + unless $nodes; + + my $l = $settings{level}; + my $sep = $settings{separator}; + if ($l == 0) { + return join($sep, @$nodes); + } elsif ($l == 1) { + return range_compress_xs($nodes, $sep); + } elsif ($l == 2) { + return _extra_compress($nodes, $sep); + } + # do some extra work to achieve the extra compression + my %domain; + my @no_domains; + for my $node (@$nodes) { + my ($host, $domain) = split '\.', $node, 2; + if ($domain) { + push @{$domain{$domain}}, $node; + } else { + push @no_domains, $host; + } + } + my @result; + if (@no_domains) { + push @result, range_compress_xs(\@no_domains, $sep); + } + for my $domain (sort keys %domain) { + my $r = _extra_compress($domain{$domain}, ","); + for ($r) { + s/\.$domain$sep/$sep/g; + s/\.$domain$//; + $_ = "{$_}" if /,/; + } + push @result, "$r.$domain"; + } + return join($sep, @result); +} + +sub _extra_compress { + my ($nodes, $sep) = @_; + my @nodes = @{$nodes}; + my %domains; + for (@nodes) { + s/^([a-z]+)(\d+)([a-z]\w+)\./$1$2.UNDOXXX$3./; + } + my $result = range_compress_xs(\@nodes, $sep); + for ($result) { + s/(\d+-\d+)\.UNDOXXX/{$1}/g; + s/(\d+)\.UNDOXXX/$1/g; + } + return $result; +} + +*compress_range = \&range_compress; +*compress = \&range_compress; + +sub nodes_parser { + my ($c, $r, $x) = @_; + my @range = (); + if (defined $r) { + push @range, $r; + } + if (defined $c) { push @range, '%' . $c; } + if (defined $x) { push @range, "-($x)" } + return range_expand_sorted(join(",", @range)); +} + +my $wanted_warnings = 0; +sub want_warnings { + my $prev = $wanted_warnings; + if (scalar @_) { + ($wanted_warnings) = @_; + want_warnings_real(@_); + } + return $prev; +} + +sub test_range { + my $w = want_warnings(0); + my ($b) = scalar(expand_range(@_)); + want_warnings($w); + return $b ? 1 : 0; +} + + +want_caching(1); +want_warnings(-t STDIN && -t STDOUT); + +# Autoload methods go after =cut, and are processed by the autosplit program. + +1; +__END__ +# Below is stub documentation for your module. You'd better edit it! + +=head1 NAME + +Seco::AwesomeRange - Perl extension for dealing with Seco ranges + +=head1 SYNOPSIS + +use Seco::AwesomeRange qw/:common/; + +my @nodes = expand_range('%ks301'); + +my $nodes = compress_range(\@nodes); + +my $same_nodes = compress_range(@nodes); + +my @sorted_nodes = sorted_expand_range('@ALL'); + +=head1 DESCRIPTION + +Do stuff with ranges. + +Expand ranges and cluster definitions. +Compress (a ref to) an array of nodes into a compact string rep. + +=head1 COMMON USAGE + +If you decide to import the most common functions into your name space: + + use Seco::AwesomeRange qw/:common/; + +That imports the functions + + expand_range + + compress_range + + sorted_expand_range + + nodes_parser + +You can also decide which functions you prefer like: + + use Seco::AwesomeRange qw/expand compress/ + + my @nodes = expand($range); + + my $nodes_repr = compress(\@nodes); + +=head2 CACHING + + By default the library will cache the results of your expansions. This is + probably what most command line utilities want. But this will have unwanted + effects if your program is a long running one. In that case you can use the + default (C) and explictily call the C function. + +=head2 WARNINGS + + By default the library will print warnings to STDERR if running under a tty, + and be quiet otherwise. You can be explicit about wanting to see the warnings using: + C + +=head1 FUNCTIONS + +=head2 compress_range + + Returns the string representation of a given list of nodes. The input can be a + reference to an array or the the list of nodes. The input doesn't have to be + sorted. + + $string = compress_range(\@nodes); + + $string = compress_range(@nodes); + +=head2 expand_range + + Expands a string representation of nodes and returns a list of nodes. The + result is not guaranteed to be in any particular order. If you care about the + result being sorted use C, otherwise use this one (it'll be faster.) + + @nodes = expand_range('%ks301-7 & @REDHAT') + +=head2 sorted_expand_range + + Same as C but the return list is sorted. + +=head2 want_caching + + want_caching(1) # caching enabled (default) + + want_caching(0) # caching is disabled + +=head2 want_warnings + + want_warnings(1) # print warnings to STDERR + + want_warnings(0) # be quiet + +=head2 get_exception + +=head2 clear_exception + +=head2 clear_caches + + Clear all caches used by librange. + + clear_caches() + +=head2 get_version + +Returns the version for librange + + my $version = Seco::AwesomeRange::get_version(); + + +=head2 expand / compress + + You can use the aliases expand/compress: + + my @nodes = Seco::AwesomeRange::expand($range); + my $repr = Seco::AwesomeRange::compress(\@nodes); + +=head2 nodes_parser + + Function provided for compatibility with the old Seco.pm module (standardNodesParser). + + my @nodes = nodes_parser(,,); + +=head1 RANGE SYNTAX + +=head2 Simple Ranges + +=head2 Union + +=head2 Intersection + +=head2 Difference + +=head2 Expand clusters + +=head2 Expand GROUPS/HOSTS + +=head2 Named functions + +=head1 SEE ALSO + +perldoc Seco::Range + +=head1 AUTHOR + +Daniel Muino, Edmuino@yahoo-inc.comE +Evan Miller, Eeam@yahoo-inc.comE + +=cut diff --git a/seco_awesomerange/source/pm_to_blib b/seco_awesomerange/source/pm_to_blib new file mode 100644 index 0000000..e69de29 diff --git a/seco_awesomerange/source/ppport.h b/seco_awesomerange/source/ppport.h new file mode 100644 index 0000000..98d08c5 --- /dev/null +++ b/seco_awesomerange/source/ppport.h @@ -0,0 +1,1096 @@ + +/* ppport.h -- Perl/Pollution/Portability Version 2.011 + * + * Automatically Created by Devel::PPPort on Thu Jul 7 04:44:41 2005 + * + * Do NOT edit this file directly! -- Edit PPPort.pm instead. + * + * Version 2.x, Copyright (C) 2001, Paul Marquess. + * Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + * This code may be used and distributed under the same license as any + * version of Perl. + * + * This version of ppport.h is designed to support operation with Perl + * installations back to 5.004, and has been tested up to 5.8.1. + * + * If this version of ppport.h is failing during the compilation of this + * module, please check if a newer version of Devel::PPPort is available + * on CPAN before sending a bug report. + * + * If you are using the latest version of Devel::PPPort and it is failing + * during compilation of this module, please send a report to perlbug@perl.com + * + * Include all following information: + * + * 1. The complete output from running "perl -V" + * + * 2. This file. + * + * 3. The name & version of the module you were trying to build. + * + * 4. A full log of the build that failed. + * + * 5. Any other information that you think could be relevant. + * + * + * For the latest version of this code, please retreive the Devel::PPPort + * module from CPAN. + * + */ + +/* + * In order for a Perl extension module to be as portable as possible + * across differing versions of Perl itself, certain steps need to be taken. + * Including this header is the first major one, then using dTHR is all the + * appropriate places and using a PL_ prefix to refer to global Perl + * variables is the second. + * + */ + + +/* If you use one of a few functions that were not present in earlier + * versions of Perl, please add a define before the inclusion of ppport.h + * for a static include, or use the GLOBAL request in a single module to + * produce a global definition that can be referenced from the other + * modules. + * + * Function: Static define: Extern define: + * newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + * + */ + + +/* To verify whether ppport.h is needed for your module, and whether any + * special defines should be used, ppport.h can be run through Perl to check + * your source code. Simply say: + * + * perl -x ppport.h *.c *.h *.xs foo/bar*.c [etc] + * + * The result will be a list of patches suggesting changes that should at + * least be acceptable, if not necessarily the most efficient solution, or a + * fix for all possible problems. It won't catch where dTHR is needed, and + * doesn't attempt to account for global macro or function definitions, + * nested includes, typemaps, etc. + * + * In order to test for the need of dTHR, please try your module under a + * recent version of Perl that has threading compiled-in. + * + */ + + +/* +#!/usr/bin/perl +@ARGV = ("*.xs") if !@ARGV; +%badmacros = %funcs = %macros = (); $replace = 0; +foreach () { + $funcs{$1} = 1 if /Provide:\s+(\S+)/; + $macros{$1} = 1 if /^#\s*define\s+([a-zA-Z0-9_]+)/; + $replace = $1 if /Replace:\s+(\d+)/; + $badmacros{$2}=$1 if $replace and /^#\s*define\s+([a-zA-Z0-9_]+).*?\s+([a-zA-Z0-9_]+)/; + $badmacros{$1}=$2 if /Replace (\S+) with (\S+)/; +} +foreach $filename (map(glob($_),@ARGV)) { + unless (open(IN, "<$filename")) { + warn "Unable to read from $file: $!\n"; + next; + } + print "Scanning $filename...\n"; + $c = ""; while () { $c .= $_; } close(IN); + $need_include = 0; %add_func = (); $changes = 0; + $has_include = ($c =~ /#.*include.*ppport/m); + + foreach $func (keys %funcs) { + if ($c =~ /#.*define.*\bNEED_$func(_GLOBAL)?\b/m) { + if ($c !~ /\b$func\b/m) { + print "If $func isn't needed, you don't need to request it.\n" if + $changes += ($c =~ s/^.*#.*define.*\bNEED_$func\b.*\n//m); + } else { + print "Uses $func\n"; + $need_include = 1; + } + } else { + if ($c =~ /\b$func\b/m) { + $add_func{$func} =1 ; + print "Uses $func\n"; + $need_include = 1; + } + } + } + + if (not $need_include) { + foreach $macro (keys %macros) { + if ($c =~ /\b$macro\b/m) { + print "Uses $macro\n"; + $need_include = 1; + } + } + } + + foreach $badmacro (keys %badmacros) { + if ($c =~ /\b$badmacro\b/m) { + $changes += ($c =~ s/\b$badmacro\b/$badmacros{$badmacro}/gm); + print "Uses $badmacros{$badmacro} (instead of $badmacro)\n"; + $need_include = 1; + } + } + + if (scalar(keys %add_func) or $need_include != $has_include) { + if (!$has_include) { + $inc = join('',map("#define NEED_$_\n", sort keys %add_func)). + "#include \"ppport.h\"\n"; + $c = "$inc$c" unless $c =~ s/#.*include.*XSUB.*\n/$&$inc/m; + } elsif (keys %add_func) { + $inc = join('',map("#define NEED_$_\n", sort keys %add_func)); + $c = "$inc$c" unless $c =~ s/^.*#.*include.*ppport.*$/$inc$&/m; + } + if (!$need_include) { + print "Doesn't seem to need ppport.h.\n"; + $c =~ s/^.*#.*include.*ppport.*\n//m; + } + $changes++; + } + + if ($changes) { + open(OUT,">/tmp/ppport.h.$$"); + print OUT $c; + close(OUT); + open(DIFF, "diff -u $filename /tmp/ppport.h.$$|"); + while () { s!/tmp/ppport\.h\.$$!$filename.patched!; print STDOUT; } + close(DIFF); + unlink("/tmp/ppport.h.$$"); + } else { + print "Looks OK\n"; + } +} +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef PERL_REVISION +# ifndef __PATCHLEVEL_H_INCLUDED__ +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define PERL_BCDVERSION ((PERL_REVISION * 0x1000000L) + (PERL_VERSION * 0x1000L) + PERL_SUBVERSION) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ + +#ifndef ERRSV +# define ERRSV perl_get_sv("@",FALSE) +#endif + +#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION <= 5)) +/* Replace: 1 */ +# define PL_Sv Sv +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_defgv defgv +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_hints hints +# define PL_na na +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfpv rsfp +# define PL_stdingv stdingv +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +/* Replace: 0 */ +#endif + +#ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +#else +# define PERL_UNUSED_DECL +#endif + +#ifndef dNOOP +# define NOOP (void)0 +# define dNOOP extern int Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef dTHR +# define dTHR dNOOP +#endif + +#ifndef dTHX +# define dTHX dNOOP +# define dTHXa(x) dNOOP +# define dTHXoa(x) dNOOP +#endif + +#ifndef pTHX +# define pTHX void +# define pTHX_ +# define aTHX +# define aTHX_ +#endif + +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif + +/* IV could also be a quad (say, a long long), but Perls + * capable of those should have IVSIZE already. */ +#if !defined(IVSIZE) && defined(LONGSIZE) +# define IVSIZE LONGSIZE +#endif +#ifndef IVSIZE +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR + +#if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +#else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +#endif +#define NUM2PTR(any,d) (any)(PTRV)(d) +#define PTR2IV(p) INT2PTR(IV,p) +#define PTR2UV(p) INT2PTR(UV,p) +#define PTR2NV(p) NUM2PTR(NV,p) +#if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +#else +# define PTR2ul(p) INT2PTR(unsigned long,p) +#endif + +#endif /* !INT2PTR */ + +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,flags) gv_stashpv(str,flags) +#endif + +#ifndef newSVpvn +# define newSVpvn(data,len) ((len) ? newSVpv ((data), (len)) : newSVpv ("", 0)) +#endif + +#ifndef newRV_inc +/* Replace: 1 */ +# define newRV_inc(sv) newRV(sv) +/* Replace: 0 */ +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef newRV_noinc +# ifdef __GNUC__ +# define newRV_noinc(sv) \ + ({ \ + SV *nsv = (SV*)newRV(sv); \ + SvREFCNT_dec(sv); \ + nsv; \ + }) +# else +# if defined(USE_THREADS) +static SV * newRV_noinc (SV * sv) +{ + SV *nsv = (SV*)newRV(sv); + SvREFCNT_dec(sv); + return nsv; +} +# else +# define newRV_noinc(sv) \ + (PL_Sv=(SV*)newRV(sv), SvREFCNT_dec(sv), (SV*)PL_Sv) +# endif +# endif +#endif + +/* Provide: newCONSTSUB */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION < 63)) + +#if defined(NEED_newCONSTSUB) +static +#else +extern void newCONSTSUB(HV * stash, char * name, SV *sv); +#endif + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) +void +newCONSTSUB(stash,name,sv) +HV *stash; +char *name; +SV *sv; +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_VERSION < 3) || ((PERL_VERSION == 3) && (PERL_SUBVERSION < 22)) + /* before 5.003_22 */ + start_subparse(), +#else +# if (PERL_VERSION == 3) && (PERL_SUBVERSION == 22) + /* 5.003_22 */ + start_subparse(0), +# else + /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +# endif +#endif + + newSVOP(OP_CONST, 0, newSVpv(name,0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif + +#endif /* newCONSTSUB */ + +#ifndef START_MY_CXT + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_VERSION < 4 || (PERL_VERSION == 4 && PERL_SUBVERSION < 68 )) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = perl_get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#else /* single interpreter */ + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif + +#endif /* START_MY_CXT */ + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef AvFILLp /* Older perls (<=5.003) lack AvFILLp */ +# define AvFILLp AvFILL +#endif + +#ifdef SvPVbyte +# if PERL_REVISION == 5 && PERL_VERSION < 7 + /* SvPVbyte does not work in perl-5.6.1, borrowed version for 5.7.3 */ +# undef SvPVbyte +# define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : my_sv_2pvbyte(aTHX_ sv, &lp)) + static char * + my_sv_2pvbyte(pTHX_ register SV *sv, STRLEN *lp) + { + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); + } +# endif +#else +# define SvPVbyte SvPV +#endif + +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_nolen(sv)) + static char * + sv_2pv_nolen(pTHX_ register SV *sv) + { + STRLEN n_a; + return sv_2pv(sv, &n_a); + } +#endif + +#ifndef get_cv +# define get_cv(name,create) perl_get_cv(name,create) +#endif + +#ifndef get_sv +# define get_sv(name,create) perl_get_sv(name,create) +#endif + +#ifndef get_av +# define get_av(name,create) perl_get_av(name,create) +#endif + +#ifndef get_hv +# define get_hv(name,create) perl_get_hv(name,create) +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef eval_pv +# define eval_pv perl_eval_pv +#endif + +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#if (PERL_VERSION > 6) || ((PERL_VERSION == 6) && (PERL_SUBVERSION >= 1)) +#define I32_CAST +#else +#define I32_CAST (I32*) +#endif + +#ifndef grok_hex +static UV _grok_hex (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_hex(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_hex(string, len, flags, result) \ + _grok_hex((string), (len), (flags), (result)) +#endif + +#ifndef grok_oct +static UV _grok_oct (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_oct(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_oct(string, len, flags, result) \ + _grok_oct((string), (len), (flags), (result)) +#endif + +#if !defined(grok_bin) && defined(scan_bin) +static UV _grok_bin (char *string, STRLEN *len, I32 *flags, NV *result) { + NV r = scan_bin(string, *len, I32_CAST len); + if (r > UV_MAX) { + *flags |= PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) *result = r; + return UV_MAX; + } + return (UV)r; +} + +# define grok_bin(string, len, flags, result) \ + _grok_bin((string), (len), (flags), (result)) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE \ + (PL_curcop == &PL_compiling ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + + +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +# define IS_NUMBER_NOT_INT 0x04 +# define IS_NUMBER_NEG 0x08 +# define IS_NUMBER_INFINITY 0x10 +# define IS_NUMBER_NAN 0x20 +#endif + +#ifndef grok_numeric_radix +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(aTHX_ sp, send) + +#define grok_numeric_radix Perl_grok_numeric_radix + +bool +Perl_grok_numeric_radix(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#if (PERL_VERSION > 6) || ((PERL_VERSION == 6) && (PERL_SUBVERSION >= 1)) + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* pre5.6.0 perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h */ +#include + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif /* PERL_VERSION */ +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif /* grok_numeric_radix */ + +#ifndef grok_number + +#define grok_number Perl_grok_number + +int +Perl_grok_number(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif /* grok_number */ + +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/seco_awesomerange/source/t/Seco-AwesomeRange.t b/seco_awesomerange/source/t/Seco-AwesomeRange.t new file mode 100644 index 0000000..bf2c91c --- /dev/null +++ b/seco_awesomerange/source/t/Seco-AwesomeRange.t @@ -0,0 +1,15 @@ +# Before `make install' is performed this script should be runnable with +# `make test'. After `make install' it should work as `perl Seco-AwesomeRange.t' + +######################### + +# change 'tests => 1' to 'tests => last_test_to_print'; + +use Test::More tests => 1; +BEGIN { use_ok('Seco::AwesomeRange') }; + +######################### + +# Insert your test code below, the Test::More module is use()ed here so read +# its man page ( perldoc Test::More ) for help writing this test script. + diff --git a/seco_awesomerange/source/t/compress_range.t b/seco_awesomerange/source/t/compress_range.t new file mode 100755 index 0000000..7a865c3 --- /dev/null +++ b/seco_awesomerange/source/t/compress_range.t @@ -0,0 +1,23 @@ +#!/usr/local/bin/perl -w +# vim:set ft=perl: -*- perl -*- + +use Test::More tests => 6; +BEGIN { use_ok('Seco::AwesomeRange',':common')}; + +ok_cmp_range('ks301000'); +ok_cmp_range('%idpproxy_yahoo1'); +ok_cmp_range('/nyn/'); +ok_cmp_range('/./'); +ok_cmp_range('1.2.3.1-1.2.3.255'); + +sub ok_cmp_range { + my $what = shift; + + my @what = expand_range($what); + my $compressed = compress_range(\@what); + my @expanded = expand_range($compressed); + #print STDERR scalar @expanded, "\n"; + ok(eq_array([sort @what], [sort @expanded]), + "expand(compress(expand($what)))==expand($what)"); +} + diff --git a/seco_awesomerange/source/t/expand_range.t b/seco_awesomerange/source/t/expand_range.t new file mode 100755 index 0000000..da63c41 --- /dev/null +++ b/seco_awesomerange/source/t/expand_range.t @@ -0,0 +1,119 @@ +#!/usr/local/bin/perl -w +# vim:set ft=perl: -*- perl -*- + +use FindBin qw/$Bin/; +use Test::More tests => 48 ; +BEGIN { use_ok('Seco::AwesomeRange', qw/expand_range range_set_altpath/) } + +my $altpath="$Bin"; +print "altpath=$altpath\n"; +range_set_altpath($altpath); + +ok_range("Parens 1", "(5,9)", qw/5 9/); +ok_range("Parens 2", "1-10,-(5,9)", qw/1 2 3 4 6 7 8 10/); + +ok_range("Numeric range no prefix", + "1-5", qw/1 2 3 4 5/); +ok_range("Numeric range commas", + "1,5", qw/1 5/); +ok_range("Simple commas", + "a1,5", qw/5 a1/); +ok_range("Implicit prefix", + "foo1-4", qw/foo1 foo2 foo3 foo4/); +ok_range("Explicit prefix", + "foo1-foo4", qw/foo1 foo2 foo3 foo4/); +ok_range("complex name ranges", + "foo1r01-foo1r04", qw/foo1r01 foo1r02 foo1r03 foo1r04/); +ok_range("Simple commas 2", + "foo,bar,baz", qw/bar baz foo/); +ok_range("Parsing commas", + "foo{1,3},bar,baz{1-3}", qw/bar baz1 baz2 baz3 foo1 foo3/); +ok_range("Multiple braces", + "foo{1,3}a{2,3}", qw/foo1a2 foo1a3 foo3a2 foo3a3/); +ok_range("Brackets and braces", + "foo{1-3}a{2,3}", qw/foo1a2 foo1a3 foo2a2 foo2a3 foo3a2 foo3a3/); +ok_range("Leading 0s", + "foo{01-03}", qw/foo01 foo02 foo03/); +ok_range("Malformed leading 0s", + "foo{01-3}", qw/foo01 foo02 foo03/); +ok_range("Simple exclude", + "foo1-4,-foo2", qw/foo1 foo3 foo4/); +ok_range("Range excludes", + "foo1-9,-foo2-8", qw/foo1 foo9/); +ok_range("Intersection", + "foo1-5,&foo3-7", qw/foo3 foo4 foo5/); +ok_range("Domains (implicit)", + "foo1-3.search", qw/foo1.search foo2.search foo3.search/); +ok_range("Domains (explicit)", + "foo1.search-foo3.search", qw/foo1.search foo2.search foo3.search/); +ok_range("Padding end", + "ks301000-3", qw/ks301000 ks301001 ks301002 ks301003/); +ok_range("Automatic dequoting", + '"ks301000-3"', qw/ks301000 ks301001 ks301002 ks301003/); +ok_range("Ignoring whitespace", ' ks301000-1 , ks30-2, foo ,bar ', + qw/bar foo ks301000 ks301001 ks30 ks31 ks32/); +ok_range("IP Ranges", '66.196.100.10-66.196.100.12', + qw/66.196.100.10 66.196.100.11 66.196.100.12/); +ok_range("IP Ranges (common prefix)", '66.196.100.10-12', + qw/66.196.100.10 66.196.100.11 66.196.100.12/); +ok_range("Regex against left side", "foo10-21,&/2/", + qw/foo12 foo20 foo21/); +ok_range("Regex against left side (set difference)", "foo10-21,-/foo.0/", + qw/foo11 foo12 foo13 foo14 foo15 foo16 foo17 foo18 foo19 foo21/); + +# Parsing clusters +ok_range("Simple cluster", '%test_cluster2', qw/kp2000 vsp2021/); +ok_range("Cluster:PART", '%test_cluster1:ADMIN', + qw/cocytus haides inferno ka1001 limbo styx/); +ok_range('Cluster:PART with $PART', '%test_cluster1:ALL', + qw/bar1 bar2 cocytus haides inferno ka1001 kp2000 limbo styx/); +ok_range('Cluster:PART with only $PART', '%test_cluster1:SMARTPOSTFIX', + qw/cocytus haides inferno ka1001 limbo styx/); +ok_range('Cluster:PART with EXCLUDE $PART', '%test_cluster1:DUMBPOSTFIX', + qw/bar1 bar2 kp2000/); +ok_range('Cluster:KEYS', '%test_cluster1:KEYS', + qw/ADMIN ALL DUMBPOSTFIX SMARTPOSTFIX STABLE CLUSTER/); +ok_range('Multiple clusters', '%test_cluster1-2', + qw/bar1 bar2 cocytus haides inferno ka1001 kp2000 limbo styx vsp2021/); +ok_range('Multiple clusters using braces', '%test_cluster{1,2}', + qw/bar1 bar2 cocytus haides inferno ka1001 kp2000 limbo styx vsp2021/); +ok_range('Cluster and intersection', '%test_cluster1,&%test_cluster2', + qw/kp2000/); +ok_range('Cluster and differences', '%test_cluster1,-%test_cluster2', + qw/bar1 bar2 cocytus haides inferno ka1001 limbo styx/); + +ok_range('Cluster:UP','%test_cluster3:UP', + qw/xy1000 xy1001 xy1002 xy1003 xy1004 xy1006 xy1007 xy1008 xy1009/); +ok_range('Cluster:DOWN','%test_cluster3:DOWN', + qw/xy1005/); +ok_range('Cluster:VIPS','%test_cluster3:VIPS', + qw/ 10.1.2.3 10.1.2.4 10.1.2.5 /); + +#ok_range('Simple Regex', '/kp/', qw/kp2000/); + +range_set_altpath(undef); + +# Test get admin function +ok_range("Simple get admin", '^@stress', qw/stress/); +ok_range("Complex get admin", '^{@stress,@hate}', qw/hate stress/); + +# just to verify that @ == %GROUPS: +eq_range('%GROUPS:AC2', '@AC2'); +eq_range('%GROUPS:AC2,-%GROUPS:ADMIN', '@AC2,-@ADMIN'); +eq_range('%GROUPS:AC2,&%GROUPS:ADMIN', '@AC2,&@ADMIN'); +eq_range('%HOSTS:stress', '@stress'); +eq_range('%HOSTS:stress,-stress', '@stress,-stress'); +eq_range('%HOSTS:stress,&stress', '@stress,&stress'); + +sub eq_range { + my ($range1, $range2) = @_; + ok(eq_array([sort(expand_range($range1))], [sort(expand_range($range2))]), + "$range1 == $range2"); +} + +sub ok_range { + my ($descr, $range, @result) = @_; + ok(eq_array([sort(expand_range($range))], [sort(@result)]), $descr) or + diag(join(",", sort(expand_range($range))), "==", + join(",", @result)); +} diff --git a/seco_awesomerange/source/t/test_cluster1/nodes.cf b/seco_awesomerange/source/t/test_cluster1/nodes.cf new file mode 100644 index 0000000..c2b218e --- /dev/null +++ b/seco_awesomerange/source/t/test_cluster1/nodes.cf @@ -0,0 +1,21 @@ +ADMIN + INCLUDE haides,inferno,ka1001 + INCLUDE limbo,styx,cocytus + +ALL + INCLUDE $ADMIN + INCLUDE kp2000 + INCLUDE bar1-2 + +SMARTPOSTFIX + INCLUDE $ADMIN + +DUMBPOSTFIX + INCLUDE $ALL + EXCLUDE $SMARTPOSTFIX + +STABLE + INCLUDE $ALL + +CLUSTER + INCLUDE $STABLE diff --git a/seco_awesomerange/source/t/test_cluster2/nodes.cf b/seco_awesomerange/source/t/test_cluster2/nodes.cf new file mode 100644 index 0000000..8ed4981 --- /dev/null +++ b/seco_awesomerange/source/t/test_cluster2/nodes.cf @@ -0,0 +1,25 @@ +LINUX + INCLUDE kp2000 + INCLUDE vsp2021 + +ALL + INCLUDE $LINUX + +############# Currernt Front End Nodes +FE + INCLUDE $ALL + +############# Nodes that billing should collect logs from +############# (Syncing is only HOURLY - if needed, quickly p4 sync on +############# ka1001, aka proxylogs1's admin) +BILLING + INCLUDE $ALL + EXCLUDE kp2000 + +############# Nodes that are currently considered stable +STABLE + INCLUDE $ALL + +CLUSTER + INCLUDE $STABLE + diff --git a/seco_awesomerange/source/t/test_cluster3/nodes.cf b/seco_awesomerange/source/t/test_cluster3/nodes.cf new file mode 100644 index 0000000..7758060 --- /dev/null +++ b/seco_awesomerange/source/t/test_cluster3/nodes.cf @@ -0,0 +1,16 @@ +ALL + INCLUDE xy1000-xy1009 + +############# Currernt Front End Nodes +FE + INCLUDE $ALL + +STABLE + INCLUDE $ALL + +CLUSTER + INCLUDE $STABLE + EXCLUDE xy1005 + + + diff --git a/seco_awesomerange/source/t/test_cluster3/vips.cf b/seco_awesomerange/source/t/test_cluster3/vips.cf new file mode 100644 index 0000000..bdf73bf --- /dev/null +++ b/seco_awesomerange/source/t/test_cluster3/vips.cf @@ -0,0 +1,3 @@ +10.1.2.3 xy1000 eth0 +10.1.2.4 xy1001 eth0 +10.1.2.5 xy1002 eth0