Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

initial commit

  • Loading branch information...
commit 71e3f97e7e20161d54e247ce00b34a945f1b5d4e 0 parents
@eam eam authored
Showing with 15,099 additions and 0 deletions.
  1. +32 −0 LICENSE.txt
  2. +729 −0 Seco_Range/Range.pm
  3. +7 −0 libcrange/index.yaml
  4. +12 −0 libcrange/root/etc/libcrange.conf.example
  5. +73 −0 libcrange/root/var/libcrange/perl/LibrangeAdminscf.pm
  6. +56 −0 libcrange/root/var/libcrange/perl/LibrangeCentcom.pm
  7. +34 −0 libcrange/root/var/libcrange/perl/LibrangeOpsdb.pm
  8. +86 −0 libcrange/root/var/libcrange/perl/LibrangeSS.pm
  9. +37 −0 libcrange/root/var/libcrange/perl/LibrangeUtils.pm
  10. +15 −0 libcrange/scripts/build
  11. +6 −0 libcrange/scripts/post.sh
  12. +6 −0 libcrange/scripts/ybuild
  13. 0  libcrange/source/AUTHORS
  14. 0  libcrange/source/ChangeLog
  15. 0  libcrange/source/INSTALL
  16. +3 −0  libcrange/source/Makefile.am
  17. 0  libcrange/source/NEWS
  18. 0  libcrange/source/README
  19. +98 −0 libcrange/source/config.h.in
  20. +117 −0 libcrange/source/configure.ac
  21. 0  libcrange/source/doc/Makefile.am
  22. +12 −0 libcrange/source/functions/Makefile.am
  23. +78 −0 libcrange/source/functions/group-mysql.c
  24. +87 −0 libcrange/source/functions/group-sqlite.c
  25. +169 −0 libcrange/source/functions/hosts-netblocks.c
  26. +13 −0 libcrange/source/functions/hosts-netblocks.h
  27. +29 −0 libcrange/source/functions/ip.c
  28. +219 −0 libcrange/source/functions/netblock.c
  29. +30 −0 libcrange/source/functions/netblock.h
  30. +809 −0 libcrange/source/functions/nodescf.c
  31. +16 −0 libcrange/source/functions/nodescf.h
  32. +127 −0 libcrange/source/functions/pgsql.c
  33. +181 −0 libcrange/source/functions/tinydns_ip.c
  34. +28 −0 libcrange/source/functions/tinydns_ip.h
  35. +104 −0 libcrange/source/functions/yst-ip-list.c
  36. 0  libcrange/source/m4/Makefile.am
  37. +74 −0 libcrange/source/perl/Libcrange.xs
  38. +25 −0 libcrange/source/perl/Makefile.PL.in
  39. +8 −0 libcrange/source/perl/build.in
  40. +86 −0 libcrange/source/perl/lib/Libcrange.pm
  41. +23 −0 libcrange/source/src/Makefile.am
  42. +120 −0 libcrange/source/src/ast.c
  43. +43 −0 libcrange/source/src/ast.h
  44. +104 −0 libcrange/source/src/config.h
  45. +25 −0 libcrange/source/src/crange.i
  46. +33 −0 libcrange/source/src/libcrange-embed.pl
  47. +446 −0 libcrange/source/src/libcrange.c
  48. +76 −0 libcrange/source/src/libcrange.h
  49. +28 −0 libcrange/source/src/main.c
  50. +227 −0 libcrange/source/src/perl_functions.c
  51. +20 −0 libcrange/source/src/perl_functions.h
  52. +430 −0 libcrange/source/src/range.c
  53. +100 −0 libcrange/source/src/range.h
  54. +110 −0 libcrange/source/src/range_compress.c
  55. +14 −0 libcrange/source/src/range_compress.h
  56. +194 −0 libcrange/source/src/range_parser.y
  57. +29 −0 libcrange/source/src/range_parser_defs.h
  58. +46 −0 libcrange/source/src/range_parts.c
  59. +25 −0 libcrange/source/src/range_parts.h
  60. +164 −0 libcrange/source/src/range_request.c
  61. +29 −0 libcrange/source/src/range_request.h
  62. +151 −0 libcrange/source/src/range_scanner.l
  63. +54 −0 libcrange/source/src/range_sort.c
  64. +14 −0 libcrange/source/src/range_sort.h
  65. +445 −0 libcrange/source/src/set.c
  66. +42 −0 libcrange/source/src/set.h
  67. +11 −0 librange/index.yaml
  68. +3 −0  librange/scripts/post.sh
  69. +41 −0 librange/source/Makefile
  70. +1,046 −0 librange/source/OCamlMakefile
  71. +62 −0 librange/source/admins.ml
  72. +4 −0 librange/source/admins.mli
  73. +19 −0 librange/source/ast.mli
  74. +2 −0  librange/source/configure
  75. +714 −0 librange/source/evaluate.ml
  76. +9 −0 librange/source/evaluate.mli
  77. +30 −0 librange/source/hosts_netblocks.ml
  78. +81 −0 librange/source/lexer.mll
  79. +179 −0 librange/source/librange.c
  80. +13 −0 librange/source/memoize.ml
  81. +55 −0 librange/source/mlrange.ml
  82. +37 −0 librange/source/netblocks.ml
  83. +3 −0  librange/source/netblocks.mli
  84. +59 −0 librange/source/netmask.ml
  85. +5 −0 librange/source/netmask.mli
  86. +52 −0 librange/source/parser.mly
  87. +18 −0 librange/source/range.h
  88. +13 −0 librange/source/range.ml
  89. +118 −0 librange/source/range_utils.ml
  90. +38 −0 librange/source/range_utils.mli
  91. +8 −0 librange/source/rip_objects_hack.sh
  92. +23 −0 librange/source/testrange-reps.c
  93. +67 −0 librange/source/testrange.c
  94. +106 −0 librange/source/tinydns.ml
  95. +8 −0 librange/source/tinydns.mli
  96. +9 −0 mod_range/index.yaml
  97. +15 −0 mod_range/root/etc/httpd/conf.d/mod_range.conf
  98. +17 −0 mod_range/scripts/build
  99. +3 −0  mod_range/scripts/post.sh
  100. +6 −0 mod_range/source/Makefile.regular
  101. +385 −0 mod_range/source/mod_yahoo_range.c
  102. +17 −0 mod_range/source/mod_yahoo_range.conf
  103. +14 −0 perl_seco_data_range/index.yaml
  104. +69 −0 perl_seco_data_range/root/usr/bin/er
  105. +186 −0 perl_seco_data_range/root/usr/share/man/man1/er.1
  106. +12 −0 perl_seco_data_range/scripts/build
  107. +13 −0 perl_seco_data_range/source/Makefile.PL
  108. +29 −0 perl_seco_data_range/source/README
  109. +378 −0 perl_seco_data_range/source/lib/Seco/Data/Range.pm
  110. +45 −0 perl_seco_data_range/source/lib/Seco/Data/Range/Simple.pm
  111. +15 −0 perl_seco_data_range/source/t/Seco-Data-Range.t
  112. +4 −0 perl_seco_libcrange/index.yaml
  113. +6 −0 perl_seco_libcrange/source/Changes
  114. +128 −0 perl_seco_libcrange/source/Libcrange.xs
  115. +8 −0 perl_seco_libcrange/source/MANIFEST
  116. +34 −0 perl_seco_libcrange/source/Makefile.PL
  117. +40 −0 perl_seco_libcrange/source/README
  118. +133 −0 perl_seco_libcrange/source/lib/Seco/Libcrange.pm
  119. +1,096 −0 perl_seco_libcrange/source/ppport.h
  120. +41 −0 perl_seco_libcrange/source/t/Seco-Libcrange.t
  121. +98 −0 seco_awesomerange/index.yaml
  122. +5 −0 seco_awesomerange/scripts/build
  123. +162 −0 seco_awesomerange/source/AwesomeRange.xs
  124. +6 −0 seco_awesomerange/source/Changes
  125. +10 −0 seco_awesomerange/source/MANIFEST
  126. +878 −0 seco_awesomerange/source/Makefile
  127. +40 −0 seco_awesomerange/source/Makefile.PL
  128. +38 −0 seco_awesomerange/source/README
  129. +55 −0 seco_awesomerange/source/const-c.inc
  130. +87 −0 seco_awesomerange/source/const-xs.inc
  131. +55 −0 seco_awesomerange/source/fallback/const-c.inc
  132. +87 −0 seco_awesomerange/source/fallback/const-xs.inc
  133. +335 −0 seco_awesomerange/source/lib/Seco/AwesomeRange.pm
  134. 0  seco_awesomerange/source/pm_to_blib
  135. +1,096 −0 seco_awesomerange/source/ppport.h
  136. +15 −0 seco_awesomerange/source/t/Seco-AwesomeRange.t
  137. +23 −0 seco_awesomerange/source/t/compress_range.t
  138. +119 −0 seco_awesomerange/source/t/expand_range.t
  139. +21 −0 seco_awesomerange/source/t/test_cluster1/nodes.cf
  140. +25 −0 seco_awesomerange/source/t/test_cluster2/nodes.cf
  141. +16 −0 seco_awesomerange/source/t/test_cluster3/nodes.cf
  142. +3 −0  seco_awesomerange/source/t/test_cluster3/vips.cf
32 LICENSE.txt
@@ -0,0 +1,32 @@
+Copyright (c) 2011, Yahoo! Inc. All rights reserved.
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the following
+conditions are met:
+
+* Redistributions of source code must retain the above
+ copyright notice, this list of conditions and the
+ following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the
+ following disclaimer in the documentation and/or other
+ materials provided with the distribution.
+
+* Neither the name of Yahoo! Inc. nor the names of its
+ contributors may be used to endorse or promote products
+ derived from this software without specific prior
+ written permission of Yahoo! Inc.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
729 Seco_Range/Range.pm
@@ -0,0 +1,729 @@
+=head1 NAME
+
+Range - deal with Seco ranges.
+
+=head1 SYNOPSIS
+
+use Range qw/:common/;
+
+my @nodes = expand_range('%ks301');
+
+my $nodes = compress_range(\@nodes);
+
+my $same_nodes = compress_range(@nodes);
+
+my @sorted_nodes = sorted_expand_range('@ALL');
+
+=head1 DESCRIPTION
+
+Do stuff with ranges.
+
+Expand ranges and cluster definitions.
+Compress (a ref to) an array of nodes into a compact string rep.
+
+Copyright (c) 2011, Yahoo! Inc. All rights reserved.
+Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+
+=cut
+
+package Seco::Range;
+
+use warnings;
+use strict;
+use Exporter;
+use IO::File;
+use Sys::Hostname;
+use Storable;
+use Carp;
+
+use constant NODE_CLUSTER => '/home/seco/candy/whoismycluster/node_cluster.dat';
+
+use vars qw(@ISA @EXPORT_OK @EXPORT %EXPORT_TAGS $VERSION $recursion);
+use vars qw($IGOR);
+use Log::Log4perl qw(:easy);
+
+@ISA = qw/Exporter/;
+@EXPORT_OK = qw/expand_range get_cluster_nodes range_set_altpath
+ compress_range sorted_expand_range nodes_parser get_expanded_clusters/;
+%EXPORT_TAGS = (
+ all => [@EXPORT_OK],
+ common => [qw/expand_range compress_range sorted_expand_range nodes_parser
+ get_expanded_clusters/]);
+@EXPORT = (); # don't pollute the namespace - use Range qw/:common/
+
+$VERSION = "1.4.1";
+$recursion = 0;
+
+#eval <<'MEMOIZE';
+#use Memoize; # hmmm caching
+#memoize('expand_range');
+#memoize('_get_cluster_keys');
+#MEMOIZE
+
+my $node_regex = qr/
+ ([-\w.]*?) # prefix
+ (\d+) # followed by the start of the range
+ (\.[-A-Za-z\d.]*[-A-Za-z]+[-A-Za-z\d.]*)? # optional domain
+/x;
+
+my ($balanced_parens, $balanced_braces);
+if ($] > 5.006) {
+ $balanced_parens = qr/
+ \(
+ (?: (?> [^( )]+ ) # no backtracking here (normal chars)
+ | # or
+ (??{ $balanced_parens }) )* # recursive matching here
+ \)
+ /x;
+
+ $balanced_braces = qr/
+ {
+ (?: (?> [^{ }]+ ) # no backtracking here (normal chars)
+ |
+ (??{ $balanced_braces }) )* # recursive matching here
+ }
+ /x;
+} else {
+ # F*ing solaris machines
+ $balanced_parens = qr/
+ \(
+ (?: (?> [^()]+ ) | \( (?> [^()]+) \)+)*
+ \)
+ /x;
+ $balanced_braces = qr/
+ {
+ (?: (?> [^{}]+ ) | { (?> [^{}]+) }+)*
+ }
+ /x;
+}
+
+my $range_re = qr@
+ (?:^|,) # beginning of the string or , is start of range
+ ( # start capturing our range
+ [^(){}/,]* # normal characters (not {} or / or ())
+ (?:
+ $balanced_parens
+ |
+ (?:-|&) \/ [^\/]+ \/ # a reg.ex.
+ | # or
+ (?:-|&) \| [^\|]+ \|
+ |
+ $balanced_braces
+ [^{},]* # followed by optional normal chars
+ )+ # one or more times
+ | # or it can be a simple thing (no braces involved)
+ (?> # never backtrack over these
+ [^(){},/]+) # normal chars
+ )@x;
+
+my $range_altpath;
+sub range_set_altpath {
+ my $new_path = shift;
+ return $range_altpath = $new_path;
+}
+
+sub _sort_nodes {
+ my $ref_nodes = shift;
+
+ my @sorted =
+ map { $_->[0] }
+ sort { $a->[1] cmp $b->[1] ||
+ $a->[3] cmp $b->[3] ||
+ $a->[2] <=> $b->[2] ||
+ $a->[0] cmp $b->[0] }
+ map { if (/^$node_regex$/) {
+ [$_, defined $1 ? $1 : "", defined $2 ? $2 : 0,
+ defined $3 ? $3 : ""]
+ } else { [$_, "", 0, ""] }
+ } @$ref_nodes;
+ return @sorted;
+}
+
+# compress_range related stuff
+sub _get_group {
+ my ($prefix, $digits, $count, $suffix) = @_;
+
+ $prefix = "" unless defined $prefix;
+ my $len_digits = length($digits);
+ my $node_fmt = "\%s\%0${len_digits}d";
+ my $group_fmt = "$node_fmt-" . substr($node_fmt, 2);
+ my $group = sprintf($group_fmt, $prefix, $digits,
+ $digits + $count);
+
+ $suffix = "" unless defined $suffix;
+ return $group . $suffix;
+}
+
+sub compress_range {
+ my @nodes = @_;
+ if (@nodes == 1 and ref $nodes[0]) {
+ @nodes = @{$nodes[0]};
+ }
+
+ unless (@nodes) {
+ _range_warn("No nodes specified.");
+ return;
+ }
+
+ @nodes = _sort_nodes(\@nodes);
+
+ my @result;
+ my ($prev_prefix, $prev_digits, $prev_suffix) = ("", undef, "");
+ my $prev_n;
+ my ($prefix, $digits, $suffix);
+ my $count = 0;
+
+ for my $n (@nodes) {
+ if ($n =~ /^$node_regex$/) {
+ ($prefix, $digits, $suffix) = ($1, $2, $3);
+ $prefix = "" unless defined $prefix;
+ $suffix = "" unless defined $suffix;
+ #print defined $prefix ? $prefix : "(undef)", " - ", defined $digits ? $digits : "(undef)", " - ", defined $suffix ? $suffix : "(undef)", "\n";
+ } else {
+ ($prefix, $digits, $suffix) = ($n, undef, undef);
+ }
+ if (defined $digits and
+ $prefix eq $prev_prefix and
+ $suffix eq $prev_suffix and
+ defined $prev_digits and
+ $digits == $prev_digits + $count + 1) {
+ $count++;
+ next;
+ }
+ if (defined $prev_n) {
+ if ($count > 0) {
+ push @result, _get_group($prev_prefix, $prev_digits, $count,
+ $prev_suffix);
+ } else {
+ push @result, $prev_n;
+ }
+ }
+
+ $prev_n = $n;
+ $prev_prefix = $prefix;
+ $prev_digits = $digits;
+ $prev_suffix = $suffix;
+ $count = 0;
+ }
+
+ if ($count > 0) {
+ push @result, _get_group($prev_prefix, $prev_digits, $count,
+ $prev_suffix);
+ } else {
+ push @result, $prev_n;
+ }
+
+ return join(",", @result);
+}
+
+sub nodes_parser {
+ my ($c, $r, $x) = @_;
+ my @range = ();
+ if (defined $r) {
+ push @range, $r;
+ }
+ if (defined $c) { push @range, '%' . $c; }
+ if (defined $x) { push @range, "-($x)" };
+
+ return sorted_expand_range(join(",", @range));
+}
+
+sub sorted_expand_range {
+ my @nodes = expand_range(@_);
+ return _sort_nodes(\@nodes);
+}
+
+my %current_clusters;
+sub get_expanded_clusters {
+ return [keys %current_clusters];
+}
+
+sub expand_range {
+ my ($ranges) = @_; return () unless defined $ranges;
+
+ local $recursion = $recursion + 1;
+ confess "Max recursion hit in expand_range"
+ if $recursion > 20;
+
+ DEBUG("expand_range($ranges)");
+ if ($recursion == 1) {
+ DEBUG("resetting current_clusters");
+ %current_clusters = () ;
+ }
+
+
+ my %nodes;
+ for ($ranges) {
+ s/\s+//g; # Get rid of spaces
+ s/^"(.*)"$/$1/; # Dequote if quoted
+
+ # Igor has no notion of expanding "hosts" vs "groups" so
+ # we're only going to use a single syntax (%HOST:)
+ #s/@([A-Za-z0-9][^,]+)/%GROUPS:$1/g; # shorthand notation for groups
+ s/@([A-Za-z0-9][^,]+)/%HOSTS:$1/g; # shorthand notation for hosts
+ }
+
+ my @ranges = $ranges =~ m/$range_re/g;
+ for my $range (@ranges) {
+ # see if it's a special range
+ if (substr($range,0,1) eq "(") {
+ my $parens_text = $range;
+ my $content = ($parens_text =~ m/^($balanced_parens)$/)[0];
+ $content = substr($content, 1, -1);
+ my @nodes = expand_range($content);
+ @nodes{@nodes} = undef;
+ } elsif ($range =~ m{^&\|([^|]+)\|} ||
+ $range =~ m{^&\/([^\/]+)\/}) {
+ # filter
+ my $regex = qr/$1/;
+ while (my ($k, $v) = each(%nodes)) {
+ delete $nodes{$k} unless $k =~ /$regex/;
+ }
+ } elsif ($range =~ m{^-\|([^|]+)\|} ||
+ $range =~ m{^-\/([^\/]+)\/}) {
+ # filter not
+ my $regex = qr/$1/;
+ while (my ($k, $v) = each(%nodes)) {
+ delete $nodes{$k} if $k =~ /$regex/;
+ }
+ } elsif ($range =~ /^(%|-|&|\^|\*)(.+)$/) {
+ # special operators that modify the rest of the range
+ my ($special_op, $rest) = ($1, $2);
+ if ($special_op eq "%") {
+ # it's a cluster
+ my @nodes = get_cluster_nodes($rest);
+ @nodes{@nodes} = undef;
+ } elsif ($special_op eq "-") {
+ # delete nodes from nodes
+ my @nodes = expand_range($rest);
+ delete @nodes{@nodes};
+ } elsif ($special_op eq "&") {
+ # intersection
+ my @common_nodes = ();
+ my @nodes = expand_range($rest);
+ for my $node (@nodes) {
+ push @common_nodes, $node if exists $nodes{$node};
+ }
+ %nodes = (); @nodes{@common_nodes} = undef;
+ } elsif ($special_op eq "^") {
+ my @nodes = expand_range($rest);
+ my @admins = _get_admins_for(@nodes);
+ @nodes{@admins} = undef;
+ } elsif ($special_op eq "*") {
+ my @nodes = expand_range($rest);
+ my @clusters = _get_clusters_for(@nodes);
+ @nodes{@clusters} = undef;
+ }
+ } else {
+ # simple range
+ if ($range =~ /^
+ $node_regex
+ - # our separator is '-'
+ \1? # the prefix again, which is optional
+ (\d+) # and the end of the range
+ ((?(3) \3 | # if the domain matched before, we want it here
+ (?:\.[-A-Za-z\d.]+)?)) # if it didn't then we can have a new
+ # one here, like foo1-3.search
+ $
+ /x)
+ {
+ my ($prefix, $start, $suf1, $end, $suf2) = ($1, $2, $3, $4, $5);
+ $prefix = "" unless defined $prefix;
+ my $suffix = "";
+ if (defined $suf1 and defined $suf2) {
+ if ($suf1 ne $suf2) {
+ warn "Different suffixes: $suf1 $suf2";
+ }
+ $suffix = $suf1;
+ } elsif (defined $suf2) {
+ $suffix = $suf2;
+ }
+ my $len = length($start);
+ # pad $end with leading characters from start so we can
+ # type 01-3 and expand that to 01,02,03 or maybe
+ # ks301000-9 for ks301000-301009
+ my $len_end = length($end);
+ $end = substr($start, 0, $len - $len_end) . $end
+ if $len_end < $len;
+ my @nodes = map {$_ = sprintf("$prefix%0${len}d$suffix", $_) }
+ ($start .. $end);
+ @nodes{@nodes} = undef;
+ } elsif ($range =~ /{/) { # }
+ my @nodes = expand_range(join(",", _expand_braces($range)));
+ @nodes{@nodes} = undef;
+ } else {
+ # single machine
+ $nodes{$range} = undef;
+ }
+ }
+ }
+ return keys %nodes;
+}
+
+sub get_cluster_nodes {
+ my $cluster = shift;
+ $cluster =~ s/HOSTS://;
+ $current_clusters{$cluster} = 1;
+ DEBUG("expanding $cluster");
+
+ my $role = $IGOR->get_role(split(/\./,$cluster,2));
+ if ($IGOR->{'use_expand_cache'}) {
+ my $members = $role->get_members();
+ my $hosts = $members->expand();
+ my $base = $members->based_on();
+ for (@$base) {
+ $current_clusters{$_} = 1;
+ }
+ return @$hosts;
+ }
+ my $members = $role->get_members()->get_range();
+ DEBUG(" $members");
+ my $nodes = [expand_range($members)];
+
+ return @$nodes;
+
+=for ignoring
+
+ my $cluster = shift;
+ my @clusters = expand_range($cluster);
+ my @result;
+
+ for $cluster (@clusters) {
+ die "Malformed cluster name $cluster"
+ unless $cluster =~ /^([-\w.]+):?([-\w.]+)?$/;
+ $cluster = $1;
+ my $part = defined($2) ? $2 : "CLUSTER";
+ my %keys = _get_cluster_keys($cluster);
+ push @result, expand_range($keys{$part});
+ }
+ return @result;
+
+=cut
+
+}
+
+{
+ my %nodes_admin;
+
+ sub _populate_nodes_admin {
+ my @admins = expand_range('%HOSTS:KEYS');
+ for my $admin (@admins) {
+ my @admin_nodes = expand_range("\%HOSTS:$admin");
+ for my $node (@admin_nodes) {
+ $nodes_admin{$node} = $admin;
+ }
+ }
+ }
+
+ sub _get_admins_for {
+ my @nodes = @_;
+ _populate_nodes_admin() unless %nodes_admin;
+ my %results;
+ for my $node (@nodes) {
+ my $admin = $nodes_admin{$node};
+ if ($admin) {
+ $results{$admin}++;
+ } else {
+ _range_warn("$node: admin not found");
+ }
+ }
+ return keys %results;
+ }
+}
+
+{
+ my $nodes_cluster;
+ sub _populate_nodes_cluster {
+ $nodes_cluster = retrieve(NODE_CLUSTER);
+ unless ($nodes_cluster) {
+ _range_warn("Can't read node_cluster.dat");
+ return;
+ }
+ }
+
+ sub _get_clusters_for {
+ my @nodes = @_;
+ _populate_nodes_cluster() unless $nodes_cluster;
+ my %results;
+ for my $node (@nodes) {
+ my $cluster = $nodes_cluster->{$node};
+ if ($cluster) {
+ $results{$cluster}++;
+ } else {
+ _range_warn("$node: cluster not found");
+ }
+ }
+ return keys %results;
+ }
+}
+
+
+sub _expand_braces {
+ my $range = shift;
+ my @todo = ($range);
+ my @results;
+ local $_;
+
+ while (@todo) {
+ $_ = shift(@todo);
+ if (/^(.*)($balanced_braces)(.*)$/) {
+ my ($pre, $braces, $post) = ($1, $2, $3);
+ my @braces = expand_range(substr($braces, 1, -1));
+ for my $elt (@braces) {
+ if ($pre =~ /{/) {
+ push @todo, "$pre$elt$post";
+ } else {
+ push @results, "$pre$elt$post";
+ }
+ }
+ } else {
+ push @results, $_;
+ }
+ }
+ return @results;
+}
+
+sub _get_cluster_keys {
+ my $cluster = shift;
+ my @lines = _read_cluster_file($cluster);
+ return unless @lines;
+
+ my (%keys, $current_key, @current_range);
+
+ for (@lines) {
+ s/#.*$//; s/\s+$//;
+ s/\$(\w+)/\%$cluster:$1/g; # Turn $MACRO into %cluster:MACRO
+ next unless /\S/;
+
+ my $joinsep;
+ if (/^\s/ && $current_key) {
+ if (/^\s+INCLUDE/) {
+ s/^\s+INCLUDE\s+//;
+ } elsif (/^\s+EXCLUDE/) {
+ s/^\s+EXCLUDE\s+(.*)/-($1)/;
+ } else {
+ die "RangeError: $_: don't know how to parse that";
+ }
+
+ s/\s+//; # strip white space
+ push @current_range, $_;
+ } else { # New Key
+ # save old key info if it exists
+ $keys{$current_key} = join(",", @current_range) if $current_key;
+
+ $current_key = $_;
+ @current_range = ();
+ }
+ }
+
+ $keys{$current_key} = join(",", @current_range) if $current_key;
+ $keys{KEYS} = join(",", keys %keys);
+ $keys{UP} = "\%${cluster}";
+ $keys{DOWN} = "\%${cluster}:ALL,-\%${cluster}:CLUSTER";
+ $keys{VIPS} = _get_cluster_vips($cluster);
+ return %keys;
+}
+
+sub _get_cluster_file {
+ my $cluster = shift;
+ my ($fh, $filename);
+ my $alt = $range_altpath || "";
+
+ ($filename) = grep( -e $_ ,
+ "$alt/$cluster/tools/conf/nodes.cf",
+ "$alt/$cluster/nodes.cf",
+ "/home/seco/tools/conf/$cluster/nodes.cf",
+ "/usr/local/gemclient/$cluster/nodes.cf");
+ $filename or _range_warn("$cluster: missing on this machine");
+ return $filename;
+}
+
+sub _read_big_file {
+ my $filename = shift;
+ open my $big_fh, '<', $filename or do {
+ _range_warn("$filename: $!");
+ return;
+ };
+
+ my @result;
+ while (<$big_fh>) {
+ if (/^\$INCLUDE\s+"([^"]+)"/) {
+ my $include = $1;
+ my $relative_dir = "./";
+ if ($include !~ m{^/}) {
+ # it's a relative PATH, prepend the dir for the cur file
+ if ($filename =~ m{^(.*/)}) {
+ $relative_dir = $1;
+ }
+ }
+
+ push @result, _read_big_file("$relative_dir$filename");
+ } else {
+ push @result, $_;
+ }
+ }
+ close $big_fh;
+ return wantarray() ? @result : join("", @result);
+}
+
+sub _read_cluster_file {
+ my $cluster = shift;
+ my $filename = _get_cluster_file($cluster);
+ my @lines = _read_big_file($filename);
+
+ # TODO: parse $INCLUDE
+ return @lines;
+}
+
+sub _range_warn {
+ my $warn = shift;
+ warn "$warn\n" if -t STDIN && -t STDOUT;
+}
+
+sub _open_cluster_vips {
+ my $cluster = shift;
+ my ($fh, $filename);
+
+ my $alt = $range_altpath || "";
+ ($filename) = grep ( -e $_,
+ "$alt/$cluster/tools/conf/vips.cf",
+ "$alt/$cluster/vips.cf",
+ "/home/seco/tools/conf/$cluster/vips.cf");
+ $filename ||= "/dev/null";
+ $fh = new IO::File "$filename", "r";
+ return $fh;
+}
+
+sub _get_cluster_vips {
+ my $cluster = shift;
+ my $fh = _open_cluster_vips($cluster);
+ my (@vips);
+
+ while (<$fh>) {
+ s/#.*$//;
+ s/\s+$//;
+ next unless /\S/;
+ my($vip) = split(/\s+/);
+ push(@vips,$vip) if ($vip =~ m/./);
+ }
+ close($fh);
+ return join(",",@vips);
+}
+
+1;
+
+__END__
+
+=head1 FUNCTIONS
+
+=head2 compress_range
+
+ $string = compress_range(\@nodes)
+
+ $string = compress_range(@nodes)
+
+=head2 expand_range
+
+ @nodes = expand_range($range) # @nodes in random order, faster
+
+=head2 sorted_expand_range
+
+ @nodes = sorted_expand_range($range) # @nodes in a nice order
+
+=head2 get_cluster_nodes
+
+ @nodes = get_cluster_nodes("ks301"); # random order
+
+=head2 range_set_altpath
+
+ range_set_altpath("/usr/local/gemclient")
+ # look for cluster definitions in the directory specified
+
+=head1 RANGE SYNTAX
+
+=head2 SIMPLE RANGES
+
+ node1,node2,node3,node4 == node1-node4 == node1-4
+
+ node1000-1099 == node1000-99 # auto pads digits to the end of the range
+
+ 1-100 # numeric only ranges
+
+ foo1-2.search.scd.yahoo.com ==
+ foo1.search.scd.yahoo.com-foo2.search.scd.yahoo.com # domain support
+
+ 209.131.40.1-209.131.40.255 == 209.131.40.1-255 # IP ranges
+
+=head2 CLUSTERS
+
+ %ks301 == nodes defined in ks301/nodes.cf - Default Section CLUSTER
+
+ %ks301:ALL == nodes defined in a specific section of ks301/nodes.cf
+
+ %ks301:VIPS == IPs in ks301/vips.cf
+
+=head2 SPECIAL CLUSTERS
+
+ %HOSTS and %GROUPS
+
+ %HOSTS:haides has all the hosts that haides is responsible for.
+
+ @haides is a shortcut for the above.
+
+ %GROUPS:ADMIN has all the machines in the group ADMIN
+
+ @ADMIN is a shortcut.
+
+
+=head2 OPERATIONS
+
+ range1,range2 == union
+
+ range1,-range2 == set difference
+
+ range1,&range2 == intersection
+
+ ^range1 == admins for the nodes in range1
+
+ range1,-(range2,range3) == () can be used for grouping
+
+ range1,&|regex| # all nodes in range1 that match regex
+
+ range1,-|regex| # all nodes in range1 that do not match regex
+
+ /regex/ == all nodes that match regex (it does matching against @ALL)
+
+ The difference between |regex| and /regex/ is that |regex| does the
+ matching against the left side of the expression, while /regex/ does
+ the matching against all nodes. Therefore
+
+ fornode.pl -r /ks30/ -l # makes sense
+
+ fornode.pl -r |ks30| -l # doesn't make sense since there's nothing to the left
+
+
+=head2 MORE ADVANCED RANGES
+
+ foo{1,3,5} == foo1,foo3,foo5
+
+ %ks30{1,3} == %ks301,%ks303
+
+ %ks301-7 == nodes in clusters ks301 to ks307
+
+ %all:KEYS == all defined sections in cluster all
+
+ %{%all} == expands all clusters in %all
+
+ %all:sc5,-({f,k}s301-7) == names for clusters in sc5 except ks301-7,fs301-7
+
+ %all:sc5,-|ks| == clusters in sc5, except those matching ks
+
+=head1 BUGS
+
+Need more docs
+
+=head1 AUTHOR
+
+Daniel Muino <dmuino@yahoo-inc.com>
+
+=cut
7 libcrange/index.yaml
@@ -0,0 +1,7 @@
+default:
+ name: libcrange
+ summary: C version of librange
+ version: '1.0.1'
+ requires:
+ - cpan-yaml-syck
+
12 libcrange/root/etc/libcrange.conf.example
@@ -0,0 +1,12 @@
+############
+#
+# Basic libcrange config file
+# Actual file gemstone managed
+#
+############
+loadmodule ip
+loadmodule yst-ip-list
+loadmodule nodescf
+perlmodule LibrangeUtils
+perlmodule LibrangeAdminscf
+
73 libcrange/root/var/libcrange/perl/LibrangeAdminscf.pm
@@ -0,0 +1,73 @@
+package LibrangeAdminscf;
+
+use Libcrange;
+use YAML::Syck;
+use warnings 'all';
+
+sub functions_provided {
+ return qw/boot_v v_boot boothosts bh/;
+}
+
+sub _get_cf {
+ my $rr = shift;
+ my $path = Libcrange::get_var($rr, "nodescf_path");
+ $path ||= "/home/seco/tools/conf";
+ my $file = "$path/admins.cf";
+ unless (-r $file) {
+ Libcrange::warn($rr, "$path/admins.cf not readable");
+ return;
+ }
+ return YAML::Syck::LoadFile($file);
+}
+
+sub boot_v {
+ my $rr = shift;
+ my $range = shift;
+
+ my $cf = _get_cf($rr);
+ return unless $cf;
+
+ my %net_bh;
+ while ( my ( $bh, $bh_cfg ) = each(%$cf) ) {
+ for my $net ( @{ $bh_cfg->{networks} } ) {
+ push @{ $net_bh{$net} }, $bh;
+ }
+ }
+
+ my @ret;
+ for my $net (@$range) {
+ $net =~ s/\A"(.*)"\z/$1/s;
+ push @ret, @{ $net_bh{$net} };
+ }
+ return @ret;
+}
+
+sub v_boot {
+ my $rr = shift;
+ my $range = shift;
+ my $cf = _get_cf();
+ return unless $cf;
+
+ my @ret;
+ for my $bh (@$range) {
+ my $bh_cfg = $cf->{$bh};
+ next unless $bh_cfg;
+
+ push @ret, map { qq("$_") } @{ $bh_cfg->{networks} };
+ }
+ return @ret;
+}
+
+sub boothosts {
+ my $rr = shift;
+ my $cf = _get_cf();
+ return keys %$cf;
+}
+
+sub bh {
+ my ($rr, $range) = @_;
+ my $what = join( ",", @{$range} );
+ return Libcrange::expand( $rr, "boot_v(vlan($what))" );
+}
+
+1;
56 libcrange/root/var/libcrange/perl/LibrangeCentcom.pm
@@ -0,0 +1,56 @@
+package LibrangeCentcom;
+
+sub functions_provided {
+ return qw/centcom_environment centcom_services centcom_facility
+ centcom_status/;
+}
+
+sub centcom_status {
+ eval "require Centcom;";
+ return () if ($@);
+ my ( $rr, $range ) = @_;
+ my @ret;
+ my $centcom = Centcom->new;
+ for my $status (@$range) {
+ push @ret, $centcom->findHosts( { status => $status } );
+ }
+ return @ret;
+}
+
+sub centcom_facility {
+ eval "require Centcom;";
+ return () if ($@);
+ my ( $rr, $range ) = @_;
+ my @ret;
+ my $centcom = Centcom->new;
+ for my $env (@$range) {
+ push @ret, $centcom->findHosts( { facility => $env } );
+ }
+ return @ret;
+}
+
+sub centcom_services {
+ eval "require Centcom;";
+ return () if ($@);
+ my ( $rr, $range ) = @_;
+ my @ret;
+ my $centcom = Centcom->new;
+ for my $env (@$range) {
+ push @ret, $centcom->findHosts( { services => $env } );
+ }
+ return @ret;
+}
+
+sub centcom_environment {
+ eval "require Centcom;";
+ return () if ($@);
+ my ( $rr, $range ) = @_;
+ my @ret;
+ my $centcom = Centcom->new;
+ for my $env (@$range) {
+ push @ret, $centcom->findHosts( { environment => $env } );
+ }
+ return @ret;
+}
+
+1;
34 libcrange/root/var/libcrange/perl/LibrangeOpsdb.pm
@@ -0,0 +1,34 @@
+package LibrangeOpsdb;
+
+use Seco::OpsDB;
+
+sub functions_provided {
+ Seco::OpsDB->connect;
+ return qw/order orders/;
+}
+
+sub orders {
+ my @result;
+ my $it = Seco::OpsDB::NodesGroup->search_where( gtype => 'groups' );
+ while ( my $order = $it->next ) {
+ push @result, $order->name;
+ }
+ return @result;
+}
+
+sub order {
+ my ( $rr, $r_orders ) = @_;
+ my @result;
+ for my $order (@$r_orders) {
+ my $it = Seco::OpsDB::NodesGroup->retrieve(
+ gname => $order,
+ gtype => 'groups'
+ );
+ while ( my $node = $it->next ) {
+ push @result, $node->name;
+ }
+ }
+ return @result;
+}
+
+1
86 libcrange/root/var/libcrange/perl/LibrangeSS.pm
@@ -0,0 +1,86 @@
+package LibrangeSS;
+
+sub functions_provided {
+ return qw/ ds_rid flatten flatten_all /;
+}
+
+sub _get_seco_root {
+ my $SECO_ROOT = "/home/seco/tools/conf/";
+ return $SECO_ROOT;
+}
+
+sub ds_rid {
+ eval "require YAML::Syck";
+ return () if ($@);
+
+ my $rr = shift;
+ my $range = shift;
+ return "one node at a time" if scalar @$range != 1;
+
+ my $SECO_ROOT = _get_seco_root();
+
+ my $node = pop @$range;
+ my $cluster = Libcrange::expand($rr, "clusters(clusters(clusters($node)))");
+
+ my $rid_file = $SECO_ROOT . $cluster . '/rid.yaml';
+ my $rid = YAML::Syck::LoadFile($rid_file);
+
+ return $rid->{$node};
+}
+
+sub rid_map {
+ my $rr = shift;
+ my $range = shift;
+ return "one cluster at a time" if scalar @$range != 1;
+
+ my $SECO_ROOT = _get_seco_root();
+
+ my $cluster = pop @$range;
+ my $rid_file = $SECO_ROOT . $cluster . '/rid.yaml';
+
+ open $fh, "$rid_file" or return "can't open rid file";
+ my $contents = join '', <$fh>;
+ close $fh;
+
+ return $contents;
+}
+
+sub flatten {
+ my $rr = shift;
+ my $range = shift;
+
+ my @ret = ();
+
+ for my $elem (@{ $range }){
+ if (my @expansion = Libcrange::expand($rr, "%$elem") ){
+ push @ret, flatten( $rr, [ $_ ] ) for @expansion;
+ }
+ else {
+ push @ret, $elem;
+ }
+ }
+
+ return @ret;
+}
+
+sub flatten_all {
+ my $rr = shift;
+ my $range = shift;
+
+ use Data::Dumper;
+ warn Dumper $range;
+
+ my @ret = ();
+
+ for my $elem (@{ $range }){
+ if (my @expansion = Libcrange::expand($rr, "%$elem") ){
+ push @ret, flatten_all( $rr, [ $_ . ":ALL" ] ) for @expansion;
+ }
+ else {
+ push @ret, $elem;
+ }
+ }
+
+ return @ret;
+}
+1;
37 libcrange/root/var/libcrange/perl/LibrangeUtils.pm
@@ -0,0 +1,37 @@
+package LibrangeUtils;
+
+sub functions_provided {
+ return qw/clean limit count even odd/;
+}
+
+sub clean {
+ my ( $rr, $range ) = @_;
+ my @result = map { s/\.inktomisearch\.com//; $_ } @$range;
+ return @result;
+}
+
+sub count {
+ my ( $rr, $range ) = @_;
+ return scalar @$range;
+}
+
+sub limit {
+ my ( $rr, $r_limit, $range ) = @_;
+ my $limit = $r_limit->[0];
+ my @range = @$range;
+ return @range[ 0 .. ( $limit - 1 ) ];
+}
+
+sub even {
+ my ( $rr, $range ) = @_;
+ my @result = grep { /[02468]\z/ms } @$range;
+ return @result;
+}
+
+sub odd {
+ my ( $rr, $range ) = @_;
+ my @result = grep { /[13579](?:\.inktomisearch\.com)?\z/ms } @$range;
+ return @result;
+}
+
+1;
15 libcrange/scripts/build
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+set -x
+
+aclocal || exit 1
+libtoolize --force || exit 1
+automake-1.9 -a || exit 1
+autoconf || exit 1
+./configure || exit 1
+make || exit 1
+make install || exit 1
+cd perl
+sh ./build || exit 1
+
6 libcrange/scripts/post.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+/sbin/ldconfig
+if [ -r /etc/httpd/conf.d/mod_range.conf ]; then
+ /etc/init.d/httpd restart
+fi
+exit 0
6 libcrange/scripts/ybuild
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+rm -rf ../BUILD
+export PATH="/home/y/bin:$PATH"
+DESTDIR=$PWD/../BUILD CFLAGS=" -DYAHOO_SOURCE " ../scripts/build
+
0  libcrange/source/AUTHORS
No changes.
0  libcrange/source/ChangeLog
No changes.
0  libcrange/source/INSTALL
No changes.
3  libcrange/source/Makefile.am
@@ -0,0 +1,3 @@
+EXTRA_DIST = reconf configure
+SUBDIRS= src functions
+
0  libcrange/source/NEWS
No changes.
0  libcrange/source/README
No changes.
98 libcrange/source/config.h.in
@@ -0,0 +1,98 @@
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you
+ don't. */
+#undef HAVE_DECL_STRERROR_R
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `crypt' library (-lcrypt). */
+#undef HAVE_LIBCRYPT
+
+/* Define to 1 if you have the `m' library (-lm). */
+#undef HAVE_LIBM
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#undef HAVE_LIBPTHREAD
+
+/* Define to 1 if you have the `z' library (-lz). */
+#undef HAVE_LIBZ
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if `stat' has the bug that it succeeds when given the
+ zero-length file name argument. */
+#undef HAVE_STAT_EMPTY_STRING_BUG
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the `strerror' function. */
+#undef HAVE_STRERROR
+
+/* Define to 1 if you have the `strerror_r' function. */
+#undef HAVE_STRERROR_R
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if `lstat' dereferences a symlink specified with a trailing
+ slash. */
+#undef LSTAT_FOLLOWS_SLASHED_SYMLINK
+
+/* Package name */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Define to 1 if strerror_r returns char *. */
+#undef STRERROR_R_CHAR_P
+
+/* Package version */
+#undef VERSION
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+ `char[]'. */
+#undef YYTEXT_POINTER
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define to `unsigned' if <sys/types.h> does not define. */
+#undef size_t
117 libcrange/source/configure.ac
@@ -0,0 +1,117 @@
+# -*- Autoconf -*-
+# Process this file with autoconf to produce a configure script.
+
+AC_PREREQ(2.59)
+AC_INIT(libcrange, 1.0, prod-eng@yahoo-inc.com)
+AC_CONFIG_SRCDIR(src/libcrange.h)
+AM_INIT_AUTOMAKE
+AM_CONFIG_HEADER([config.h])
+
+AC_DEFINE_UNQUOTED(PACKAGE, $PACKAGE, [Package name])
+AC_DEFINE_UNQUOTED(VERSION, $VERSION, [Package version])
+
+# Checks for programs.
+AC_PROG_YACC
+AM_PROG_LEX
+AC_PROG_CC
+AC_LANG_C
+AC_PROG_INSTALL
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+AC_PROG_LIBTOOL
+
+perl_default="/usr/local/bin/perl"
+AC_ARG_ENABLE([perl], AC_HELP_STRING([--enable-perl=PERL],
+ [Use this perl binary to build libcrange's perl
+ functionality.
+ (Default = /usr/local/bin/perl)]),
+ PERL=$enable_perl,PERL=$perl_default)
+AC_DEFUN([AC_CHECK_PERL],[
+AC_MSG_CHECKING(perl in $PERL)
+if test [ -f "$PERL" ]
+then
+ AC_SUBST(PERL)
+ AC_MSG_RESULT(yes)
+ AC_MSG_CHECKING(perl includes)
+ PERL_CFLAGS=`${PERL} -MExtUtils::Embed -e ccopts`
+ AC_SUBST(PERL_CFLAGS)
+ AC_MSG_RESULT($PERL_CFLAGS)
+ AC_MSG_CHECKING(perl libraries)
+ PERL_LIBS=`${PERL} -MExtUtils::Embed -e ldopts`
+ AC_SUBST(PERL_LIBS)
+ AC_MSG_RESULT($PERL_LIBS)
+else
+ AC_MSG_ERROR([${PERL} not found])
+fi
+])
+
+AC_DEFUN([AC_CHECK_PCRE],[
+AC_PATH_PROG(pcreconfig,pcre-config)
+if test [ -z "$pcreconfig" ]
+then
+ AC_MSG_ERROR([pcre-config executable not found])
+else
+ AC_MSG_CHECKING(pcre includes)
+ PCRE_CFLAGS=`${pcreconfig} --cflags`
+ AC_MSG_RESULT($PCRE_CFLAGS)
+ AC_SUBST(PCRE_CFLAGS)
+ AC_MSG_CHECKING(pcre libraries)
+ PCRE_LIBS=`${pcreconfig} --libs`
+ AC_MSG_RESULT($PCRE_LIBS)
+ AC_SUBST(PCRE_LIBS)
+fi
+])
+
+AC_DEFUN([AC_CHECK_APR],[
+AC_PATH_PROG(aprconfig,apr-config)
+if test [ -z "$aprconfig" ]
+then
+ AC_PATH_PROG(aprconfig, apr-1-config)
+fi
+if test [ -z "$aprconfig" ]
+then
+ AC_MSG_ERROR([apr-config/apr-1-config executable not found])
+else
+ AC_MSG_CHECKING(apr includes)
+ APR_CFLAGS=`${aprconfig} --cflags --cppflags --includes`
+ AC_MSG_RESULT($APR_CFLAGS)
+ AC_SUBST(APR_CFLAGS)
+ AC_MSG_CHECKING(apr libraries)
+ APR_LIBS=`${aprconfig} --link-libtool --libs`
+ APR_LIBS_LD=`${aprconfig} --link-ld --libs`
+ AC_MSG_RESULT($APR_LIBS)
+ AC_SUBST(APR_LIBS)
+ AC_SUBST(APR_LIBS_LD)
+fi
+])
+
+AC_CHECK_PERL
+AC_CHECK_PCRE
+AC_CHECK_APR
+
+AC_CHECK_LIB([crypt], [crypt_r], [], [exit 1])
+AC_CHECK_LIB([m], [sin], [], [exit 1])
+AC_CHECK_LIB([pthread], [pthread_create], [], [exit 1])
+AC_CHECK_LIB([z], [zlibVersion], [], [exit 1])
+
+# Checks for header files.
+AC_HEADER_STDC
+AC_CHECK_HEADERS([stdlib.h string.h unistd.h])
+
+# Checks for typedefs, structures, and compiler characteristics.
+AC_C_CONST
+AC_TYPE_SIZE_T
+
+# Checks for library functions.
+AC_FUNC_STAT
+AC_FUNC_STRERROR_R
+AC_CHECK_FUNCS([strerror])
+
+AC_CONFIG_FILES([Makefile
+ doc/Makefile
+ m4/Makefile
+ src/Makefile
+ perl/Makefile.PL
+ perl/build
+ functions/Makefile])
+AC_OUTPUT
0  libcrange/source/doc/Makefile.am
No changes.
12 libcrange/source/functions/Makefile.am
@@ -0,0 +1,12 @@
+AM_CFLAGS = -Wall -DLIBCRANGE_FUNCDIR=\"$(pkglibdir)\" -I../src @PCRE_CFLAGS@ @APR_CFLAGS@
+AM_LDFLAGS = -module -L../src -lcrange @PCRE_LIBS@ @APR_LIBS@
+
+pkglib_LTLIBRARIES = yst-ip-list.la ip.la nodescf.la
+
+nodescf_la_SOURCES = nodescf.c
+yst_ip_list_la_SOURCES = yst-ip-list.c netblock.c tinydns_ip.c \
+ hosts-netblocks.c
+ip_la_SOURCES = ip.c tinydns_ip.c
+
+
+
78 libcrange/source/functions/group-mysql.c
@@ -0,0 +1,78 @@
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <mysql/mysql.h>
+#include <apr_strings.h>
+
+#include "set.h"
+#include "libcrange.h"
+#include "range.h"
+
+const char** functions_provided(libcrange* lr)
+{
+ static const char* functions[] = {"group", 0};
+ return functions;
+}
+
+range* rangefunc_group(range_request* rr, range** r)
+{
+ range *ret;
+ const char **members;
+ int i;
+ MYSQL *conn;
+ MYSQL_ROW row;
+ MYSQL_RES *res;
+ apr_pool_t* pool = range_request_pool(rr);
+ libcrange* lr = range_request_lr(rr);
+
+ ret = range_new(rr);
+ members = range_get_hostnames(pool, r[0]);
+
+ if(!(conn = (MYSQL *)libcrange_get_cache(lr, "mysql:nodes"))) {
+ const char* mysql_user = libcrange_getcfg(lr, "mysqluser");
+ const char* mysql_db = libcrange_getcfg(lr, "mysqldb");
+ const char* mysql_passwd = libcrange_getcfg(lr, "mysqlpasswd");
+
+ conn = mysql_init(NULL);
+ mysql_real_connect(conn, "docking", mysql_user, mysql_passwd,
+ mysql_db, 0, NULL, 0);
+ libcrange_set_cache(lr, "mysql:nodes", conn);
+ }
+ for(i = 0; members[i]; i++) { /* for each gemgroup */
+ int all = strcmp(members[i], "ALL") == 0;
+
+ if (all) {
+ const char* query = "select name from nodes";
+ if (mysql_query(conn, query)) {
+ fprintf(stderr, "query: %s failed: %s\n",
+ query, mysql_error(conn));
+ return range_new(rr);
+ }
+ } else {
+ const char* query = apr_psprintf(pool,
+ "select range from tags where name='%s'",
+ members[i]);
+ if (mysql_query(conn, query)) {
+ fprintf(stderr, "query: %s failed: %s\n",
+ query, mysql_error(conn));
+ return range_new(rr);
+ }
+ }
+ res = mysql_store_result(conn);
+ assert(res);
+ while ((row = mysql_fetch_row(res)) != NULL) {
+ range* this_group;
+ const char* result = row[0];
+ if (all) {
+ range_add(ret, result);
+ } else {
+ this_group = do_range_expand(rr, result);
+ set_union_inplace(ret->nodes, this_group->nodes);
+ }
+ }
+ mysql_free_result(res);
+ }
+
+ return ret;
+}
87 libcrange/source/functions/group-sqlite.c
@@ -0,0 +1,87 @@
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sqlite3.h>
+
+#include "set.h"
+#include "libcrange.h"
+#include "range.h"
+
+const char** functions_provided(libcrange* lr)
+{
+ static const char* functions[] = {"group", 0};
+ return functions;
+}
+
+#define ALL_NODES_SQL "select name from nodes"
+#define RANGE_FROM_TAGS "select range from tags where name=?"
+
+range* rangefunc_group(range_request* rr, range** r)
+{
+ range* ret;
+ const char** members;
+ int i, err;
+ sqlite3* db;
+ sqlite3_stmt* tag_stmt;
+ sqlite3_stmt* all_nodes_stmt;
+ apr_pool_t* pool = range_request_pool(rr);
+ libcrange* lr = range_request_lr(rr);
+
+ ret = range_new(rr);
+ members = range_get_hostnames(pool, r[0]);
+
+ if (!(db = libcrange_get_cache(lr, "sqlite:nodes"))) {
+ const char* sqlite_db_path = libcrange_getcfg(lr, "sqlitedb");
+ if (!sqlite_db_path) sqlite_db_path = DEFAULT_SQLITE_DB;
+
+ err = sqlite3_open(sqlite_db_path, &db);
+ if (err != SQLITE_OK) {
+ fprintf(stderr, "%s: %s\n", sqlite_db_path, sqlite3_errmsg(db));
+ return ret;
+ }
+
+ libcrange_set_cache(lr, "sqlite:nodes", db);
+ }
+
+
+ /* prepare our selects */
+ err = sqlite3_prepare(db, ALL_NODES_SQL, strlen(ALL_NODES_SQL),
+ &all_nodes_stmt, NULL);
+ if (err != SQLITE_OK) {
+ fprintf(stderr, "%s: %s\n", ALL_NODES_SQL, sqlite3_errmsg(db));
+ abort();
+ }
+
+ err = sqlite3_prepare(db, RANGE_FROM_TAGS, strlen(RANGE_FROM_TAGS),
+ &tag_stmt, NULL);
+ assert(err == SQLITE_OK);
+
+ /* for each group */
+ for (i = 0; members[i]; ++i) {
+ sqlite3_stmt* stmt;
+ if (strcmp(members[i], "ALL") == 0) {
+ stmt = all_nodes_stmt;
+ } else {
+ stmt = tag_stmt;
+ /* bind the current group name */
+ sqlite3_bind_text(tag_stmt, 1, members[i], strlen(members[i]), SQLITE_STATIC);
+ }
+
+ while (sqlite3_step(stmt) == SQLITE_ROW) {
+ range* this_group;
+ const char* result = (const char*)sqlite3_column_text(stmt, 0);
+ if (stmt == all_nodes_stmt) {
+ range_add(ret, result);
+ } else {
+ this_group = do_range_expand(rr, result);
+ set_union_inplace(ret->nodes, this_group->nodes);
+ }
+ }
+ sqlite3_reset(stmt);
+ }
+ sqlite3_finalize(all_nodes_stmt);
+ sqlite3_finalize(tag_stmt);
+
+ return ret;
+}
169 libcrange/source/functions/hosts-netblocks.c
@@ -0,0 +1,169 @@
+#include "hosts-netblocks.h"
+#include "netblock.h"
+#include "set.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#define HOSTS_NETBLOCK_CACHE "yst-ip-list:hosts_netblocks"
+#define HOSTS_DC_CACHE "yst-ip-list:hosts_dc"
+#define DC_HOSTS_CACHE "yst-ip-list:dc_hosts"
+#define NETBLOCK_HOSTS_CACHE "yst-ip-list:netblocks_hosts"
+
+static void init_caches(libcrange* lr)
+{
+ const char* default_domain;
+ apr_pool_t* pool = libcrange_get_pool(lr);
+ range_request* rr;
+
+ /* Create a netblock -> hosts mapping */
+ ip_host** all_hosts = tinydns_all_ip_hosts(lr, pool);
+ set* hn = set_new(pool, 100000); /* hosts networks */
+ set* hd = set_new(pool, 100000); /* hosts datacenter */
+ set* nh = set_new(pool, 1500) ; /* network hosts */
+ set* dh = set_new(pool, 0); /* datacenter hosts */
+
+ assert(all_hosts);
+ rr = range_request_new(lr, pool);
+ default_domain = libcrange_get_default_domain(lr);
+ while (*all_hosts) {
+ char short_hostname[512];
+ int len_hostname;
+ char net_key[32];
+ set_element* elt;
+ range* r;
+ ip_host* iph = *all_hosts++;
+
+ const netblock* block;
+ const net_colo* nc = netcolo_for_ip(lr, iph->ip);
+ if (!nc) continue;
+
+ strncpy(short_hostname, iph->hostname, sizeof short_hostname);
+ short_hostname[sizeof short_hostname - 1] = '\0';
+ len_hostname = strlen(short_hostname);
+ if (short_hostname[len_hostname - 1] == '.')
+ /* remove '.' at the end adjusting len */
+ short_hostname[--len_hostname] = '\0';
+
+ if (default_domain) {
+ /* remove default_domain */
+ int len_domain = strlen(default_domain);
+ if (len_hostname > len_domain) {
+ if (strcmp(&short_hostname[len_hostname - len_domain],
+ default_domain) == 0)
+ short_hostname[len_hostname - len_domain - 1] = '\0';
+ }
+ }
+
+ block = nc->net;
+ set_add(hn, short_hostname, (void*)block);
+ set_add(hn, iph->ip->str, (void*)block);
+ set_add(hd, short_hostname, (void*)(nc->colo));
+ set_add(hd, iph->ip->str, (void*)(nc->colo));
+
+ elt = set_get(nh, netblock_key(block, net_key, sizeof net_key));
+ if (!elt) {
+ r = range_new(rr);
+ set_add(nh, net_key, r);
+ }
+ else
+ r = elt->data;
+ range_add(r, short_hostname);
+
+ elt = set_get(dh, nc->colo);
+ if (!elt) {
+ r = range_new(rr);
+ set_add(dh, nc->colo, r);
+ }
+ else
+ r = elt->data;
+ range_add(r, short_hostname);
+ }
+ libcrange_set_cache(lr, HOSTS_NETBLOCK_CACHE, hn);
+ libcrange_set_cache(lr, NETBLOCK_HOSTS_CACHE, nh);
+ libcrange_set_cache(lr, DC_HOSTS_CACHE, dh);
+ libcrange_set_cache(lr, HOSTS_DC_CACHE, hd);
+}
+
+static set* hosts_netblocks(libcrange* lr)
+{
+ set* hn = libcrange_get_cache(lr, HOSTS_NETBLOCK_CACHE);
+ if (!hn) {
+ init_caches(lr);
+ hn = libcrange_get_cache(lr, HOSTS_NETBLOCK_CACHE);
+ }
+ return hn;
+}
+
+static set* netblock_hosts(libcrange* lr)
+{
+ set* nh = libcrange_get_cache(lr, NETBLOCK_HOSTS_CACHE);
+ if (!nh) {
+ init_caches(lr);
+ nh = libcrange_get_cache(lr, NETBLOCK_HOSTS_CACHE);
+ }
+ return nh;
+}
+
+static set* datacenter_hosts(libcrange* lr)
+{
+ set* dh = libcrange_get_cache(lr, DC_HOSTS_CACHE);
+ if (!dh) {
+ init_caches(lr);
+ dh = libcrange_get_cache(lr, DC_HOSTS_CACHE);
+ }
+ return dh;
+}
+
+static set* hosts_dc(libcrange* lr)
+{
+ set* hd = libcrange_get_cache(lr, HOSTS_DC_CACHE);
+ if (!hd) {
+ init_caches(lr);
+ hd = libcrange_get_cache(lr, HOSTS_DC_CACHE);
+ }
+ return hd;
+}
+
+range* hosts_in_netblock(range_request* rr, const char* netblock_key)
+{
+ libcrange* lr = range_request_lr(rr);
+ set* nh = netblock_hosts(lr);
+ set_element* elt = set_get(nh, netblock_key);
+ if (elt)
+ return elt->data;
+
+ range_request_warn_type(rr, "NETBLOCK_NOT_FOUND", netblock_key);
+ return range_new(rr);
+}
+
+range* hosts_in_dc(range_request* rr, const char* dc)
+{
+ libcrange* lr = range_request_lr(rr);
+ set* dh = datacenter_hosts(lr);
+ set_element* elt = set_get(dh, dc);
+ if (elt)
+ return elt->data;
+
+ range_request_warn_type(rr, "DC_NOT_FOUND", dc);
+ return range_new(rr);
+}
+
+const netblock* netblock_for_host(range_request* rr, const char* host)
+{
+ libcrange* lr = range_request_lr(rr);
+ set* hn = hosts_netblocks(lr);
+ set_element* block = set_get(hn, host);
+
+ if (block) return block->data;
+ return NULL;
+}
+
+const char* dc_for_host(range_request* rr, const char* host)
+{
+ libcrange* lr = range_request_lr(rr);
+ set* hd = hosts_dc(lr);
+ set_element* dc = set_get(hd, host);
+ if (dc) return dc->data;
+ return NULL;
+}
13 libcrange/source/functions/hosts-netblocks.h
@@ -0,0 +1,13 @@
+#ifndef HOSTS_NETBLOCKS_H
+#define HOSTS_NETBLOCKS_H
+
+#include "libcrange.h"
+#include "range.h"
+#include "netblock.h"
+
+range* hosts_in_netblock(range_request* rr, const char* netblock_key);
+const netblock* netblock_for_host(range_request* rr, const char* host);
+range* hosts_in_dc(range_request* rr, const char* dc);
+const char* dc_for_host(range_request* rr, const char* host);
+
+#endif
29 libcrange/source/functions/ip.c
@@ -0,0 +1,29 @@
+#include "libcrange.h"
+#include "range.h"
+#include "tinydns_ip.h"
+
+const char** functions_provided(libcrange* lr)
+{
+ static const char* functions[] = {"ip", 0};
+ return functions;
+}
+
+range* rangefunc_ip(range_request* rr, range** r)
+{
+ range* ret;
+ const char** members;
+ int i;
+ ip* ip;
+ apr_pool_t* pool = range_request_pool(rr);
+
+ ret = range_new(rr);
+ members = range_get_hostnames(pool, r[0]);
+ for (i = 0; members[i]; i++) {
+ ip = tinydns_get_ip(rr, members[i]);
+ if (ip)
+ range_add(ret, ip->str);
+ else
+ range_request_warn_type(rr, "NOTINYDNS", members[i]);
+ }
+ return ret;
+}
219 libcrange/source/functions/netblock.c
@@ -0,0 +1,219 @@
+#include "netblock.h"
+#include "set.h"
+#include <pcre.h>
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <ctype.h>
+#include <apr_strings.h>
+
+#define IMASK(n) (0 - (1 << (32 - n)))
+#define NETMASK_RE "^\"?(\\d+\\.\\d+\\.\\d+\\.\\d+)/(\\d+)\"?$"
+#define IP_RE "^\\d+\\.\\d+\\.\\d+\\.\\d+$"
+#define TWO_FIELDS_RE "^(\\S+)\\s+(\\S+)"
+
+#define NET_COLO_CACHE "net:netkey-netcolo"
+#define COLO_NET_CACHE "net:colo-netblock"
+
+static pcre* netmask_re = 0;
+static pcre* ip_re = 0;
+static pcre* two_fields_re = 0;
+
+static void compile_regexes()
+{
+ if (!netmask_re) {
+ int err_offset;
+ const char* error;
+ netmask_re = pcre_compile(NETMASK_RE, 0, &error, &err_offset, NULL);
+ assert(netmask_re);
+
+ ip_re = pcre_compile(IP_RE, 0, &error, &err_offset, NULL);
+ assert(ip_re);
+
+ two_fields_re = pcre_compile(TWO_FIELDS_RE, 0, &error, &err_offset, NULL);
+ assert(two_fields_re);
+ }
+}
+
+static set* read_netblocks(libcrange* lr)
+{
+ set* result;
+ set* colo_nets;
+
+ FILE* fp;
+ char line[4096];
+ int line_no;
+ apr_pool_t* pool;
+ range_request* rr;
+
+ if ((result = libcrange_get_cache(lr, NET_COLO_CACHE)) != 0)
+ return result;
+
+ pool = libcrange_get_pool(lr);
+ compile_regexes();
+ result = set_new(pool, 0);
+ colo_nets = set_new(pool, 0);
+
+ fp = fopen(YST_IP_LIST, "r");
+ if (!fp) {
+ fprintf(stderr, "%s: %s", YST_IP_LIST,
+ strerror(errno));
+ return set_new(pool, 0);
+ }
+
+ line_no = 0;
+ rr = range_request_new(lr, pool);
+ while (fgets(line, sizeof line, fp)) {
+ int count;
+ int ovector[30];
+ int n;
+ char* p = line;
+ const char* colo;
+ const char* netblock_str;
+ char* key;
+ set_element* elt;
+ range* r;
+ netblock* net;
+ net_colo* nc = apr_palloc(pool, sizeof(net_colo));
+
+ ++line_no;
+ p = line;
+ while (*p && isspace(*p)) ++p;
+ if (*p == '#') continue;
+ n = strlen(p);
+ if (p[n - 1] != '\n') {
+ fprintf(stderr, "%s: line %d is too long.\n",
+ YST_IP_LIST, line_no);
+ fclose(fp);
+ return set_new(pool, 0);
+ }
+ count = pcre_exec(two_fields_re, NULL, p, n,
+ 0, 0, ovector, 30);
+ if (count < 3) continue;
+
+ colo = &p[ovector[2]];
+ p[ovector[3]] = '\0';
+
+ netblock_str = &p[ovector[4]];
+ p[ovector[5]] = '\0';
+
+ net = netblock_from_string(pool, netblock_str);
+ nc->net = net;
+ nc->colo = apr_pstrdup(pool, colo);
+
+ key = apr_psprintf(pool, "%x/%d", net->base, net->bits);
+ set_add(result, key, nc);
+ elt = set_get(colo_nets, colo);
+ if (!elt) {
+ r = range_new(rr);
+ r->quoted = 1;
+ set_add(colo_nets, colo, r);
+ }
+ else
+ r = elt->data;
+ range_add(r, net->str);
+ }
+ fclose(fp);
+
+ libcrange_set_cache(lr, COLO_NET_CACHE, colo_nets);
+ libcrange_set_cache(lr, NET_COLO_CACHE, result);
+ return result;
+}
+
+net_colo* netcolo_for_ip(libcrange* lr, const ip* node_ip)
+{
+ int bits;
+ set* netblocks;
+ unsigned bin_ip;
+
+ assert(lr);
+ assert(node_ip);
+ netblocks = read_netblocks(lr);
+ bin_ip = node_ip->binary;
+ for (bits=32; bits > 0; --bits) {
+ char netmask[32];
+ set_element* elt;
+ unsigned base = bin_ip & IMASK(bits);
+ sprintf(netmask, "%x/%d", base, bits);
+ elt = set_get(netblocks, netmask);
+ if (elt)
+ return elt->data;
+ }
+ return NULL;
+}
+
+char* netblock_key(const netblock* block, char* buf, size_t n)
+{
+ snprintf(buf, n, "%x/%d", block->base, block->bits);
+ buf[n - 1] = '\0';
+ return buf;
+}
+
+const char* netblock_to_str(const netblock* block)
+{
+ assert(block);
+ return block->str;
+}
+
+netblock* netblock_from_string(apr_pool_t* pool, const char* netmask)
+{
+ int count;
+ int ovector[30];
+ const char* base;
+ const char* bits;
+ int len = strlen(netmask);
+ netblock* result = apr_palloc(pool, sizeof(netblock));
+
+ compile_regexes();
+ count = pcre_exec(netmask_re, NULL, netmask, len,
+ 0, 0, ovector, 30);
+ if (count > 0) {
+ pcre_get_substring(netmask, ovector, count, 1, &base);
+ pcre_get_substring(netmask, ovector, count, 2, &bits);
+
+ result->base = str2ip(base);
+ result->bits = atoi(bits);
+
+ result->str = apr_pstrdup(pool, netmask);
+ }
+ else {
+ char* repr = apr_palloc(pool, len + 4);
+ count = pcre_exec(ip_re, NULL, netmask, len,
+ 0, 0, ovector, 30);
+ /* FIXME: deal with errors */
+ if (count <= 0) {
+ printf("ERROR: [%s] trying to parse a netblock\n", netmask);
+ abort();
+ }
+
+ result->bits = 32;
+ pcre_get_substring(netmask, ovector, count, 0, &base);
+ result->base = str2ip(base);
+ pcre_free_substring(base);
+
+ strcpy(repr, netmask);
+ strcat(repr, "/32");
+ result->str = repr;
+ }
+
+ return result;
+}
+
+range* netblocks_for_dc(range_request* rr, const char* dc)
+{
+ set* colo_nets;
+ set_element* elt;
+ libcrange* lr = range_request_lr(rr);
+
+ read_netblocks(lr); /* make sure the caches are up-to-date */
+
+ colo_nets = libcrange_get_cache(lr, COLO_NET_CACHE);
+ elt = set_get(colo_nets, dc);
+ if (elt)
+ return elt->data;
+ else {
+ range_request_warn_type(rr, "NO_DC", dc);
+ return range_new(rr);
+ }
+}
30 libcrange/source/functions/netblock.h
@@ -0,0 +1,30 @@
+#ifndef NETBLOCK_H
+#define NETBLOCK_H
+
+#include "libcrange.h"
+#include "range.h"
+#include "tinydns_ip.h"
+
+#define YST_IP_LIST "/etc/yst-ip-list"
+
+typedef struct netblock
+{
+ unsigned base;
+ int bits;
+ const char* str;
+} netblock;
+
+typedef struct net_colo
+{
+ netblock* net;
+ const char* colo;
+} net_colo;
+
+net_colo* netcolo_for_ip(libcrange* lr, const ip* node_ip);
+const char* netblock_to_str(const netblock* block);
+netblock* netblock_from_string(apr_pool_t* pool, const char* netmask);
+char* netblock_key(const netblock* block, char* buf, size_t n);
+
+range* netblocks_for_dc(range_request* lr, const char* dc);
+
+#endif
809 libcrange/source/functions/nodescf.c
@@ -0,0 +1,809 @@
+#include <assert.h>
+#include <pcre.h>
+#include <stdio.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <sys/stat.h>
+#include <apr_strings.h>
+#include <apr_tables.h>
+#include "libcrange.h"
+#include "range.h"
+
+static const char* nodescf_path = "/home/seco/tools/conf";
+
+const char** functions_provided(libcrange* lr)
+{
+ static const char* functions[] = {"mem", "cluster", "clusters",
+ "group", "get_admin", "get_cluster",
+ "get_groups", "has", "allclusters", 0 };
+
+ /* initialize our path to the nodes database */
+
+ /* First try env variable */
+ const char* altpath = getenv( "LIBCRANGE_NODESCF_PATH" );
+
+ /* Next try variable from config file */
+ if( ! altpath )
+ altpath = libcrange_getcfg(lr, "nodescf_path");
+
+ /* If no alternative was specified, keep the default */
+ if (altpath)
+ nodescf_path = altpath;
+
+ return functions;
+}
+
+
+typedef struct cache_entry
+{
+ time_t mtime;
+ apr_pool_t* pool;
+ set* sections;
+} cache_entry;
+
+static set* _get_ignore_set(range_request* rr)
+{
+ char line[32768];
+ apr_pool_t* pool = range_request_pool(rr);
+ const char* ignore_path = apr_psprintf(pool, "%s/all/IGNORE", nodescf_path);
+ set* ret = set_new(pool, 0);
+ FILE* fp = fopen(ignore_path, "r");
+ if (!fp) return ret;
+
+ while (fgets(line, sizeof line, fp) != NULL) {
+ int len;
+ char* p;
+ line[sizeof line - 1] = '\0';
+ len = strlen(line);
+
+ if (!len) continue;
+ if (line[len - 1] == '\n')
+ line[--len] = '\0';
+
+ for (p = line; *p; ++p)
+ if (*p == '#') {
+ *p = '\0';
+ len = strlen(line);
+ break;
+ }
+
+ if (!len) continue;
+
+ for (p = &line[len - 1]; isspace(*p); --p) {
+ --len;
+ *p = '\0';
+ }
+ if (len <= 0) continue;
+
+ set_add(ret, line, 0);
+ }
+ fclose(fp);
+
+ return ret;
+}
+
+#define INCLUDE_RE "^\\s+INCLUDE\\s+(.+)"
+#define EXCLUDE_RE "^\\s+EXCLUDE\\s+(.+)"
+
+static pcre* include_re = NULL;
+static pcre* exclude_re = NULL;
+
+static char* _substitute_dollars(apr_pool_t* pool,
+ const char* cluster, const char* line)
+{
+ static char buf[262144];
+ char* dst = buf;
+ int len = strlen(cluster);
+ int in_regex = 0;
+ char c;
+ assert(line);
+ assert(cluster);
+
+ while ((c = *line) != '\0') {
+ if (!in_regex && c == '$') {
+ strcpy(dst, "cluster(");
+ dst += sizeof("cluster(") - 1;
+ strcpy(dst, cluster);
+ dst += len;
+ *dst++ = ':';
+ c = *++line;
+ while (isalnum(c) || c == '_') {
+ *dst++ = c;
+ c = *++line;
+ }
+ *dst++ = ')';
+ }
+ else if (c == '/') {
+ in_regex = !in_regex;
+ *dst++ = *line++;
+ }
+ else {
+ *dst++ = *line++;
+ }
+ }
+ *dst = '\0';
+ return buf;
+}
+
+char* _join_elements(apr_pool_t* pool, char sep, set* the_set)
+{
+ set_element** members = set_members(the_set);
+ set_element** p = members;
+ int total;
+ char* result;
+ char* p_res;
+
+ total = 0;
+ while (*p) {
+ total += strlen((*p)->name) + 1;
+ ++p;
+ }
+ if (!total) return NULL;
+
+ p = members;
+ p_res = result = apr_palloc(pool, total);
+ while (*p) {
+ int len = strlen((*p)->name);
+ strcpy(p_res, (*p)->name);
+ ++p;
+ p_res += len;
+ *p_res++ = sep;
+ }
+
+ *--p_res = '\0';
+ return result;
+}
+
+static pcre* vips_re = NULL;
+
+typedef struct vips
+{
+ time_t mtime;
+ apr_pool_t* pool;
+ set* vips;
+ set* viphosts;
+} vips;
+
+static vips* _empty_vips(range_request* rr)
+{
+ apr_pool_t* pool = range_request_pool(rr);
+ vips* v = apr_palloc(pool, sizeof(*v));
+ v->vips = v->viphosts = set_new(pool, 0);
+ return v;
+}
+
+static vips* _parse_cluster_vips(range_request* rr, const char* cluster)
+{
+ struct stat st;
+ int ovector[30];
+ char line[32768];
+ int line_no;
+ FILE* fp;
+ apr_pool_t* req_pool = range_request_pool(rr);
+ apr_pool_t* lr_pool = range_request_lr_pool(rr);
+ libcrange* lr = range_request_lr(rr);
+ set* cache = libcrange_get_cache(lr, "nodescf:cluster_vips");
+ const char* vips_path = apr_psprintf(req_pool, "%s/%s/vips.cf",
+ nodescf_path, cluster);
+ vips* v;
+
+ if (!cache) {
+ cache = set_new(lr_pool, 0);
+ libcrange_set_cache(lr, "nodescf:cluster_vips", cache);
+ }
+
+ if (stat(vips_path, &st) == -1) {
+ range_request_warn_type(rr, "NOVIPS", cluster);
+ return _empty_vips(rr);
+ }
+
+ v = set_get_data(cache, vips_path);
+ if (!v) {
+ v = apr_palloc(lr_pool, sizeof(struct vips));
+ apr_pool_create(&v->pool, lr_pool);
+ v->vips = set_new(v->pool, 0);
+ v->viphosts = set_new(v->pool, 0);
+ v->mtime = st.st_mtime;
+ set_add(cache, vips_path, v);
+ }
+ else {
+ time_t cached_mtime = v->mtime;
+ if (cached_mtime != st.st_mtime) {
+ apr_pool_clear(v->pool);
+ v->vips = set_new(v->pool, 0);
+ v->viphosts = set_new(v->pool, 0);
+ v->mtime = st.st_mtime;
+ }
+ else /* current cached copy is good */
+ return v;
+ }
+
+ /* create / update the current cached copy */
+ fp = fopen(vips_path, "r");
+ if (!fp) {
+ range_request_warn_type(rr, "NOVIPS", cluster);
+ return _empty_vips(rr);
+ }