Browse files

Initial import of CM3

  • Loading branch information...
0 parents commit cad8c7ef5a39f3b2de8427dbe1467e3c3864a42d @outofjungle committed Aug 9, 2012
Showing with 527,078 additions and 0 deletions.
  1. +129 −0 LICENSE.Artistic
  2. +15 −0 README
  3. +279 −0 azsync/azsync
  4. +86 −0 azsync/azsync-builddir
  5. +240 −0 azsync/recursive-zsync
  6. +23 −0 builder/bin/doozer
  7. +45 −0 builder/conf/builder.conf
  8. +393 −0 builder/lib/Chisle/Builder/Engine.pm
  9. +124 −0 builder/lib/Chisle/Builder/Engine/Actuate.pm
  10. +306 −0 builder/lib/Chisle/Builder/Engine/Checkout.pm
  11. +248 −0 builder/lib/Chisle/Builder/Engine/Generator.pm
  12. +240 −0 builder/lib/Chisle/Builder/Engine/Packer.pm
  13. +278 −0 builder/lib/Chisle/Builder/Engine/Walrus.pm
  14. +166 −0 builder/lib/Chisle/Builder/Group.pm
  15. +42 −0 builder/lib/Chisle/Builder/Group/Base.pm
  16. +190 −0 builder/lib/Chisle/Builder/Group/CMDBNode.pm
  17. +87 −0 builder/lib/Chisle/Builder/Group/CMDBNodeGroup.pm
  18. +45 −0 builder/lib/Chisle/Builder/Group/Host.pm
  19. +158 −0 builder/lib/Chisle/Builder/Group/Roles.pm
  20. +1,193 −0 builder/lib/Chisle/Builder/Overmind.pm
  21. +87 −0 builder/lib/Chisle/Builder/Overmind/Host.pm
  22. +166 −0 builder/lib/Chisle/Builder/Overmind/Metafile.pm
  23. +148 −0 builder/lib/Chisle/Builder/Overmind/TransformSet.pm
  24. +228 −0 builder/lib/Chisle/Builder/Raw.pm
  25. +39 −0 builder/lib/Chisle/Builder/Raw/Base.pm
  26. +85 −0 builder/lib/Chisle/Builder/Raw/Filesystem.pm
  27. +61 −0 builder/lib/Chisle/Builder/Raw/Hash.pm
  28. +165 −0 builder/lib/Chisle/Builder/Raw/HostList.pm
  29. +78 −0 builder/lib/Chisle/Builder/Raw/Roles.pm
  30. +112 −0 builder/lib/Chisle/Builder/Raw/UserGroup.pm
  31. +318 −0 builder/lib/Chisle/Builder/ZooKeeper/Base.pm
  32. +409 −0 builder/lib/Chisle/Builder/ZooKeeper/Leader.pm
  33. +180 −0 builder/lib/Chisle/Builder/ZooKeeper/Worker.pm
  34. +85 −0 builder/lib/Chisle/CheckoutPack.pm
  35. +311 −0 builder/lib/Chisle/CheckoutPack/Staged.pm
  36. +114 −0 builder/lib/Chisle/Loadable.pm
  37. +126 −0 builder/lib/Chisle/RawFile.pm
  38. +105 −0 builder/lib/Chisle/Tag.pm
  39. +616 −0 builder/lib/Chisle/Transform.pm
  40. +478 −0 builder/lib/Chisle/TransformModel.pm
  41. +99 −0 builder/lib/Chisle/TransformModel/Group.pm
  42. +122 −0 builder/lib/Chisle/TransformModel/Homedir.pm
  43. +248 −0 builder/lib/Chisle/TransformModel/Iptables.pm
  44. +113 −0 builder/lib/Chisle/TransformModel/Passwd.pm
  45. +330 −0 builder/lib/Chisle/TransformModel/PasswdLike.pm
  46. +22 −0 builder/lib/Chisle/TransformModel/Sudoers.pm
  47. +35 −0 builder/lib/Chisle/TransformModel/Sysctl.pm
  48. +114 −0 builder/lib/Chisle/TransformModel/Text.pm
  49. +90 −0 builder/libexec/doozer/doozer-build-generate
  50. +78 −0 builder/libexec/doozer/doozer-build-pack
  51. +259 −0 builder/libexec/doozer/doozer-checkout
  52. +161 −0 builder/libexec/doozer/doozer-cluster
  53. +103 −0 builder/libexec/doozer/doozer-describe
  54. +118 −0 builder/libexec/doozer/doozer-log
  55. +26 −0 builder/libexec/doozer/doozer-overmind
  56. +100 −0 builder/libexec/doozer/doozer-review
  57. +59 −0 builder/libexec/doozer/doozer-run
  58. +435 −0 builder/libexec/doozer/doozer-sync-roles
  59. +42 −0 builder/libexec/doozer/doozer-var
  60. +32 −0 builder/libexec/doozer/doozer-zkclean
  61. +111 −0 builder/libexec/doozer/doozer-zkmaster
  62. +71 −0 builder/libexec/doozer/doozer-zkworker
  63. +69 −0 builder/t/builder-stupid-group.t
  64. +30 −0 builder/t/checkout-dupe-tags.t
  65. +52 −0 builder/t/checkout-dupe-transforms.t
  66. +98 −0 builder/t/checkout-get-transforms-symlinked.t
  67. +100 −0 builder/t/checkout-get-transforms.t
  68. +33 −0 builder/t/checkout-module-files.t
  69. +80 −0 builder/t/checkout-raw.t
  70. +47 −0 builder/t/checkoutpack-unicode.t
  71. +135 −0 builder/t/checkoutpack.t
  72. +319 −0 builder/t/engine-build.t
  73. +67 −0 builder/t/engine-lock.t
  74. +58 −0 builder/t/engine.t
  75. +47 −0 builder/t/files/builder.conf
  76. +2 −0 builder/t/files/configs.1/modules/homedir/module.conf
  77. +1 −0 builder/t/files/configs.1/modules/passwd/files/base
  78. +5 −0 builder/t/files/configs.1/modules/passwd/module.conf
  79. +1 −0 builder/t/files/configs.1/modules/passwd/scripts/passwd.1
  80. +2 −0 builder/t/files/configs.1/raw/cmdb_usergroup/bad_users
  81. 0 builder/t/files/configs.1/raw/cmdb_usergroup/empty_users
  82. +4 −0 builder/t/files/configs.1/raw/cmdb_usergroup/good_users
  83. +3 −0 builder/t/files/configs.1/raw/group_role/bad.test.role
  84. BIN builder/t/files/configs.1/raw/group_role/binary.test.role
  85. BIN builder/t/files/configs.1/raw/group_role/binary.test.role2
  86. 0 builder/t/files/configs.1/raw/group_role/empty.test.role
  87. +338,252 −0 builder/t/files/configs.1/raw/group_role/ginormous
  88. +3 −0 builder/t/files/configs.1/raw/group_role/good.test.role
  89. +169,126 −0 builder/t/files/configs.1/raw/group_role/huge
  90. +1 −0 builder/t/files/configs.1/raw/motd
  91. +3 −0 builder/t/files/configs.1/raw/passwd
  92. +2 −0 builder/t/files/configs.1/raw/rawtest
  93. +1 −0 builder/t/files/configs.1/raw/rawtest2
  94. +1 −0 builder/t/files/configs.1/raw/unicode
  95. BIN builder/t/files/configs.1/raw/yahoo.png
  96. +2 −0 builder/t/files/configs.1/tags.1/TAG B
  97. +2 −0 builder/t/files/configs.1/tags.1/taga
  98. +1 −0 builder/t/files/configs.1/tags.2/GLOBAL
  99. +2 −0 builder/t/files/configs.1/tags.2/TAG B
  100. +2 −0 builder/t/files/configs.1/tags.2/taga
  101. +2 −0 builder/t/files/configs.1/tags/notag
  102. +18 −0 builder/t/files/configs.1/transforms/DEFAULT
  103. +20 −0 builder/t/files/configs.1/transforms/DEFAULT_TAIL
  104. +2 −0 builder/t/files/configs.1/transforms/func/>'a(b) & c"
  105. +6 −0 builder/t/files/configs.1/transforms/func/BADBAD
  106. +14 −0 builder/t/files/configs.1/transforms/func/BAR
  107. +5 −0 builder/t/files/configs.1/transforms/func/BINARY
  108. +5 −0 builder/t/files/configs.1/transforms/func/FOO
  109. +2 −0 builder/t/files/configs.1/transforms/func/INVALID
  110. +5 −0 builder/t/files/configs.1/transforms/func/MODULE_BUNDLE
  111. +12 −0 builder/t/files/configs.1/transforms/func/QUX
  112. +18 −0 builder/t/files/configs.1/transforms/func/UNICODE
  113. +2 −0 builder/t/files/configs.1/transforms/host/bar1
  114. +2 −0 builder/t/files/configs.1/transforms/host/not.a.host
  115. +1 −0 builder/t/files/configs.readraw/notraw/notfile
  116. +1 −0 builder/t/files/configs.readraw/raw/file.link
  117. +1 −0 builder/t/files/configs.readraw/raw/notfile.link
  118. +1 −0 builder/t/files/configs.readraw/raw/notraw.link/notfile
  119. +12 −0 builder/t/files/configs.readraw/raw/resolv.link
  120. +1 −0 builder/t/files/configs.readraw/raw/subdir/file
  121. +111 −0 builder/t/files/expected.yaml
  122. +5 −0 builder/t/files/l4p.conf
  123. +7 −0 builder/t/files/passwd/group
  124. +9 −0 builder/t/files/passwd/group.2
  125. +22 −0 builder/t/files/passwd/group.3
  126. +4 −0 builder/t/files/passwd/group.4
  127. +5 −0 builder/t/files/passwd/passwd
  128. +6 −0 builder/t/files/passwd/passwd.2
  129. +15 −0 builder/t/files/passwd/passwd.3
  130. +4 −0 builder/t/files/passwd/passwd.4
  131. +97 −0 builder/t/files/ranges.yaml
  132. +1 −0 builder/t/files/ws/f/00/00000000000000000000000000000000000001
  133. +1 −0 builder/t/files/ws/f/00/00000000000000000000000000000000000002
  134. +1 −0 builder/t/files/ws/f/00/00000000000000000000000000000000000003
  135. 0 builder/t/files/ws/f/01/00000000000000000000000000000000000001
  136. 0 builder/t/files/ws/f/01/00000000000000000000000000000000000002
  137. 0 builder/t/files/ws/f/01/00000000000000000000000000000000000003
  138. +35 −0 builder/t/generator-binary-files.t
  139. +55 −0 builder/t/generator-context-readraw.t
  140. +59 −0 builder/t/generator-generate.t
  141. +61 −0 builder/t/generator-mixed-transforms.t
  142. +29 −0 builder/t/generator-readraw.t
  143. +41 −0 builder/t/generator-unicode.t
  144. +13 −0 builder/t/generator.t
  145. +15 −0 builder/t/group-opsdb.t
  146. +14 −0 builder/t/group-roles.t
  147. +148 −0 builder/t/group.t
  148. +38 −0 builder/t/lib/AnyEventDevelCoverHack.pm
  149. +103 −0 builder/t/lib/ChilseTest/Engine.pm
  150. +53 −0 builder/t/lib/ChilseTest/FakeFunc.pm
  151. +117 −0 builder/t/lib/ChilseTest/Mock/ZooKeeper.pm
  152. +160 −0 builder/t/lib/ChilseTest/Overmind.pm
  153. +139 −0 builder/t/lib/ChilseTest/Transform.pm
  154. +43 −0 builder/t/metrics.t
  155. +58 −0 builder/t/overmind-build-bug5312559.t
  156. +70 −0 builder/t/overmind-build-generate-error.t
  157. +56 −0 builder/t/overmind-build-pack-error.t
  158. +38 −0 builder/t/overmind-build-update-hosts.t
  159. +57 −0 builder/t/overmind-build-update-raw.t
  160. +83 −0 builder/t/overmind-build-update-transforms.t
  161. +85 −0 builder/t/overmind-build.t
  162. +12 −0 builder/t/overmind.t
  163. +120 −0 builder/t/packer-pack.t
  164. +166 −0 builder/t/packer-sanity.t
  165. +12 −0 builder/t/packer.t
  166. +59 −0 builder/t/raw-expiration.t
  167. +77 −0 builder/t/raw-fetch.t
  168. +117 −0 builder/t/raw-filesystem-more.t
  169. +39 −0 builder/t/raw-filesystem.t
  170. +103 −0 builder/t/raw-hostlist.t
  171. +155 −0 builder/t/raw-keykeeper.t
  172. +112 −0 builder/t/raw-opsdb-usergroup.t
  173. +51 −0 builder/t/raw-roles.t
  174. +75 −0 builder/t/raw-validate.t
  175. +94 −0 builder/t/raw.t
  176. +61 −0 builder/t/rawfile.t
  177. +129 −0 builder/t/tag.t
  178. +87 −0 builder/t/transform-action-group-add.t
  179. +39 −0 builder/t/transform-action-group-give_me_all_groups.t
  180. +101 −0 builder/t/transform-action-group-include.t
  181. +93 −0 builder/t/transform-action-homedir-addkey.t
  182. +52 −0 builder/t/transform-action-homedir-append.t
  183. +58 −0 builder/t/transform-action-homedir-clearkey.t
  184. +25 −0 builder/t/transform-action-homedir-truncate.t
  185. +143 −0 builder/t/transform-action-iptables.t
  186. +87 −0 builder/t/transform-action-passwd-add.t
  187. +62 −0 builder/t/transform-action-passwd-append.t
  188. +66 −0 builder/t/transform-action-passwd-chsh.t
  189. +24 −0 builder/t/transform-action-passwd-dedupe.t
  190. +38 −0 builder/t/transform-action-passwd-delete.t
  191. +31 −0 builder/t/transform-action-passwd-deletere.t
  192. +53 −0 builder/t/transform-action-passwd-give_me_all_users.t
  193. +45 −0 builder/t/transform-action-passwd-include.t
  194. +54 −0 builder/t/transform-action-passwd-invokefor.t
  195. +66 −0 builder/t/transform-action-passwd-remove.t
  196. +66 −0 builder/t/transform-action-passwd-replace.t
  197. +73 −0 builder/t/transform-action-passwd-replacere.t
  198. +24 −0 builder/t/transform-action-passwd-sortuid.t
  199. +87 −0 builder/t/transform-action-passwd-srsadd.t
  200. +31 −0 builder/t/transform-action-passwd-truncate.t
  201. +38 −0 builder/t/transform-action-passwd-use.t
  202. +45 −0 builder/t/transform-action-sudoers-add.t
  203. +47 −0 builder/t/transform-action-sudoers-invokefor.t
  204. +46 −0 builder/t/transform-action-sysctl-set.t
  205. +58 −0 builder/t/transform-action-text-append.t
  206. +52 −0 builder/t/transform-action-text-appendexact.t
  207. +58 −0 builder/t/transform-action-text-appendunique.t
  208. +16 −0 builder/t/transform-action-text-dedupe.t
  209. +29 −0 builder/t/transform-action-text-delete.t
  210. +35 −0 builder/t/transform-action-text-deletere.t
  211. +35 −0 builder/t/transform-action-text-include.t
  212. +110 −0 builder/t/transform-action-text-invokefor.t
  213. +28 −0 builder/t/transform-action-text-nop.t
  214. +52 −0 builder/t/transform-action-text-prepend.t
  215. +53 −0 builder/t/transform-action-text-remove.t
  216. +59 −0 builder/t/transform-action-text-replace.t
  217. +172 −0 builder/t/transform-action-text-replacere.t
  218. +16 −0 builder/t/transform-action-text-truncate.t
  219. +22 −0 builder/t/transform-action-text-unlink.t
  220. +35 −0 builder/t/transform-action-text-use.t
  221. +160 −0 builder/t/transform-meta.t
  222. +78 −0 builder/t/transform-module-conf-actions.t
  223. +84 −0 builder/t/transform-module-conf-default-file.t
  224. +100 −0 builder/t/transform-module-conf-macros.t
  225. +12 −0 builder/t/transform-module-conf-model.t
  226. +164 −0 builder/t/transform-order.t
  227. +73 −0 builder/t/transform-raw-needed.t
  228. +198 −0 builder/t/transform.t
  229. +41 −0 builder/t/walrus-get-buckets.t
  230. +115 −0 builder/t/walrus-require-group.t
  231. +62 −0 builder/t/walrus-tags-global.t
  232. +48 −0 builder/t/walrus-tags-require.t
  233. +62 −0 builder/t/walrus-tags.t
  234. +17 −0 builder/t/walrus.t
  235. +60 −0 builder/t/zookeeper-ads.t
  236. +30 −0 builder/t/zookeeper-config.t
  237. +55 −0 builder/t/zookeeper-part.t
  238. +87 −0 builder/t/zookeeper-rebalance.t
  239. +39 −0 builder/t/zookeeper-register.t
  240. +31 −0 builder/t/zookeeper-report.t
  241. +26 −0 builder_dt/Makefile
  242. +4 −0 builder_dt/service/doozer_build/log/run
  243. +2 −0 builder_dt/service/doozer_build/run
  244. +4 −0 builder_dt/service/doozer_zkclean/log/run
  245. +2 −0 builder_dt/service/doozer_zkclean/run
  246. +4 −0 builder_dt/service/doozer_zkmaster/log/run
  247. +2 −0 builder_dt/service/doozer_zkmaster/run
  248. +4 −0 builder_dt/service/doozer_zkworker/log/run
  249. +2 −0 builder_dt/service/doozer_zkworker/run
  250. +106 −0 builder_ws/conf/apache.conf
  251. +108 −0 builder_ws/lib/Chisel/BuilderWeb/ClientValidate.pm
Sorry, we could not display the entire diff because too many files (1,038) changed.
129 LICENSE.Artistic
@@ -0,0 +1,129 @@
+ The "Artistic License"
+
+ Preamble
+
+The intent of this document is to state the conditions under which a
+Package may be copied, such that the Copyright Holder maintains some
+semblance of artistic control over the development of the package,
+while giving the users of the package the right to use and distribute
+the Package in a more-or-less customary fashion, plus the right to make
+reasonable modifications.
+
+Definitions:
+
+ "Package" refers to the collection of files distributed by the
+ Copyright Holder, and derivatives of that collection of files
+ created through textual modification.
+
+ "Standard Version" refers to such a Package if it has not been
+ modified, or has been modified in accordance with the wishes
+ of the Copyright Holder as specified below.
+
+ "Copyright Holder" is whoever is named in the copyright or
+ copyrights for the package.
+
+ "You" is you, if you're thinking about copying or distributing
+ this Package.
+
+ "Reasonable copying fee" is whatever you can justify on the
+ basis of media cost, duplication charges, time of people involved,
+ and so on. (You will not be required to justify it to the
+ Copyright Holder, but only to the computing community at large
+ as a market that must bear the fee.)
+
+ "Freely Available" means that no fee is charged for the item
+ itself, though there may be fees involved in handling the item.
+ It also means that recipients of the item may redistribute it
+ under the same conditions they received it.
+
+1. You may make and give away verbatim copies of the source form of the
+Standard Version of this Package without restriction, provided that you
+duplicate all of the original copyright notices and associated disclaimers.
+
+2. You may apply bug fixes, portability fixes and other modifications
+derived from the Public Domain or from the Copyright Holder. A Package
+modified in such a way shall still be considered the Standard Version.
+
+3. You may otherwise modify your copy of this Package in any way, provided
+that you insert a prominent notice in each changed file stating how and
+when you changed that file, and provided that you do at least ONE of the
+following:
+
+ a) place your modifications in the Public Domain or otherwise make them
+ Freely Available, such as by posting said modifications to Usenet or
+ an equivalent medium, or placing the modifications on a major archive
+ site such as uunet.uu.net, or by allowing the Copyright Holder to include
+ your modifications in the Standard Version of the Package.
+
+ b) use the modified Package only within your corporation or organization.
+
+ c) rename any non-standard executables so the names do not conflict
+ with standard executables, which must also be provided, and provide
+ a separate manual page for each non-standard executable that clearly
+ documents how it differs from the Standard Version.
+
+ d) make other distribution arrangements with the Copyright Holder.
+
+4. You may distribute the programs of this Package in object code or
+executable form, provided that you do at least ONE of the following:
+
+ a) distribute a Standard Version of the executables and library files,
+ together with instructions (in the manual page or equivalent) on where
+ to get the Standard Version.
+
+ b) accompany the distribution with the machine-readable source of
+ the Package with your modifications.
+
+ c) give non-standard executables non-standard names, and clearly
+ document the differences in manual pages (or equivalent), together
+ with instructions on where to get the Standard Version.
+
+ d) make other distribution arrangements with the Copyright Holder.
+
+5. You may charge a reasonable copying fee for any distribution of this
+Package. You may charge any fee you choose for support of this
+Package. You may not charge a fee for this Package itself. However,
+you may distribute this Package in aggregate with other (possibly
+commercial) programs as part of a larger (possibly commercial) software
+distribution provided that you do not advertise this Package as a
+product of your own. You may embed this Package's interpreter within
+an executable of yours (by linking); this shall be construed as a mere
+form of aggregation, provided that the complete Standard Version of the
+interpreter is so embedded.
+
+6. The scripts and library files supplied as input to or produced as
+output from the programs of this Package do not automatically fall
+under the copyright of this Package, but belong to whoever generated
+them, and may be sold commercially, and may be aggregated with this
+Package. If such scripts or library files are aggregated with this
+Package via the so-called "undump" or "unexec" methods of producing a
+binary executable image, then distribution of such an image shall
+neither be construed as a distribution of this Package nor shall it
+fall under the restrictions of Paragraphs 3 and 4, provided that you do
+not represent such an executable image as a Standard Version of this
+Package.
+
+7. C subroutines (or comparably compiled subroutines in other
+languages) supplied by you and linked into this Package in order to
+emulate subroutines and variables of the language defined by this
+Package shall not be considered part of this Package, but are the
+equivalent of input as in Paragraph 6, provided these subroutines do
+not change the language in any way that would cause it to fail the
+regression tests for the language.
+
+8. Aggregation of this Package with a commercial distribution is always
+permitted provided that the use of this Package is embedded; that is,
+when no overt attempt is made to make this Package's interfaces visible
+to the end user of the commercial distribution. Such use shall not be
+construed as a distribution of this Package.
+
+9. The name of the Copyright Holder may not be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+ The End
+
+
15 README
@@ -0,0 +1,15 @@
+All file copyrights licensed under the Perl Artistic License.
+See the accompanying LICENSE.Artistic files for terms.
+
+Required Dependencies:
+
+curl - http://curl.haxx.se/
+openssl - http://openssl.org
+zsync - http://zsync.moria.org.uk/
+
+Optional Dependencies:
+
+libzookeeper - perl bindings for zookeeper
+ http://zookeeper.apache.org
+apache2 - http://httpd.apache.org
+
279 azsync/azsync
@@ -0,0 +1,279 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+#!/usr/local/bin/perl -w
+
+=pod
+
+azsync is the client. It expects a
+setup on the server in the form provided by azsync-builddir
+
+Quick steps:
+
+ mkdir foo
+ echo somedata > foo/bar
+ cd foo
+ generate_manifest -o azsync.manifest.json -d .
+ # edit out azsync.manifest.json ##FIXME
+ azsync-mkdir -m azsync.manifest.json
+ # now place foo up on some httpd
+ azsync --from http://url/to/foo --to localdir
+
+The manifest must refer to relative paths with no . or ..
+
+It will zsync the manifest, then iterate the manifest to
+discover remaining files relative to the manifest's base url.
+
+Logic should be the same as arsync, with a cp -Rl and
+version directory management, as well as external validation.
+
+=cut
+
+use warnings;
+use strict;
+
+use Getopt::Long;
+use File::Path;
+
+
+my %defaults = (
+ 'cpargs' => [ '-Rl' ],
+ );
+my %opt = (
+ 'h|help' => 'Print this message',
+ 'from=s' => 'Url to sync from, eg http://host/dir/',
+ 'to=s' => 'Directory to sync to',
+ 'clean=s' => 'Remove all orphans (does not rsync)', # FIXME
+ 'md5-verify' => 'Require /to/azsync.manifest.json to be verified against md5 key',
+ 'sha1-verify' => 'Require /to/azsync.manifest.json to be verified against sha1 key', # not implemented
+ 'crc32-verify' => 'Require /to/azsync.manifest.json to be verified against crc32 key', # not implemented
+ 'external-verify=s@' => 'External verification',
+ 'noaction' => 'Take no action, only print plan',
+ 'rzsync=s' => 'specify alternate path to recursive-zsync',
+ 'curl' => 'Use curl instead of LWP to fetch Manifest. Searches PATH for curl binary',
+ 'cpargs=s@' => "Args for cp (on bsd4 you probably want -Rp instead of -Rl)",
+ 'sslcert=s' => "Client certificate file (SSL)",
+ 'sslkey=s' => "Private key file name (SSL)",
+ 'interface=s' => "Attempt to use this interface name, IP address or host name as the source of http connections",
+ 'request-timeout=i' => "Timeout for each http request made",
+ );
+
+my %o;
+die "bad options" unless GetOptions( \%o, keys %opt );
+
+# set any defaults in options hash that aren't set
+for my $k (keys %defaults) {
+ $o{$k} ||= $defaults{$k}
+}
+
+die "need --from url" unless $o{from};
+die "need --to directory" unless $o{to};
+
+### FIXME CLEAN
+die "--clean replaces --to, pick only one" if $o{clean} and $o{to}; # FIXME
+$o{to} = $o{clean} if $o{clean};
+### END FIXME
+
+my $to = $o{to};
+my $from = $o{from};
+my $noact = $o{noaction};
+my $rzsync = $o{rzsync};
+my $cpargs = $o{cpargs};
+$rzsync ||= "/bin/recursive-zsync";
+
+my @rzsync_opts;
+push @rzsync_opts, "--curl" if $o{'curl'};
+push @rzsync_opts, "--sslcert", $o{'sslcert'} if $o{'sslcert'};
+push @rzsync_opts, "--sslkey", $o{'sslkey'} if $o{'sslkey'};
+push @rzsync_opts, "--interface", $o{'interface'} if $o{'interface'};
+push @rzsync_opts, "--request-timeout", $o{'request-timeout'} if $o{'request-timeout'};
+
+print "[Not taking any action, only printing plan]\n" if $noact;
+
+# If to/current exists but isn't a symlink (is a file, is a
+# regular directory), clean it up.
+if (-e "$to/current" && !-l "$to/current") {
+ warn "$to/current exists but is not a symlink, deleting\n";
+ rmtree("$to/current") unless($noact);
+}
+
+# If to/previous exists but isn't a symlink (is a file, is a
+# regular directory), clean it up.
+if (-e "$to/previous" && !-l "$to/previous") {
+ warn "$to/previous exists but is not a symlink, deleting\n";
+ rmtree("$to/previous") unless($noact);
+}
+
+# Figure out where the current and previous symlinks are pointing.
+my $current = readlink "$to/current";
+my $previous = readlink "$to/previous";
+
+# If the current symlink exists and tries to link outside the
+# working directory, clean it up.
+if (defined($current) && $current =~ /\//) {
+ warn "$to/current links to something with a slash, deleting\n";
+ unlink "$to/current" unless($noact);
+ $current = undef;
+}
+
+# If the previous symlink exists and tries to link outside the
+# working directory, clean it up.
+if (defined($previous) && $previous =~ /\//) {
+ warn "$to/previous links to something with a slash, deleting\n";
+ unlink "$to/previous" unless($noact);
+ $previous = undef;
+}
+
+# If the current symlink exists but points to something that
+# doesn't exist, clean it up.
+if (defined($current) && !stat("$to/$current")) {
+ warn "$to/current is a dangling symlink, deleting\n";
+ unlink "$to/current" unless($noact);
+ $current = undef;
+}
+
+# If the previous symlink exists but points to something that
+# doesn't exist, clean it up.
+if (defined($previous) && !stat("$to/$previous")) {
+ warn "$to/previous is a dangling symlink, deleting\n";
+ unlink "$to/previous" unless($noact);
+ $previous = undef;
+}
+# If the previous symlink exists but the current symlink doesn't,
+# clean up the previous symlink.
+if (defined($previous) && !defined($current)) {
+ warn "$to/previous exists but not $to/current, deleting\n";
+ unlink "$to/previous" unless($noact);
+ $previous = undef;
+}
+
+# If they asked for cleaning, we'll go through and delete directories
+# that aren't pointed to by either the current or previous symlinks.
+if ($o{clean}) {
+ # Read in the contents of the directory
+ opendir(DIR, $o{clean});
+ my @files = readdir(DIR);
+ closedir(DIR);
+
+ foreach my $sub (@files) {
+ # skip dotfiles
+ next if ($sub =~ /^\./);
+ # Skip if the current symlink points to this
+ next if (defined $current && $sub eq $current);
+ # Skip if the previous symlink points to this
+ next if (defined $previous && $sub eq $previous);
+ # Skip if it's not a directory
+ next unless (-d "$to/$sub");
+ # Skip if it's a symlink (-d on a symlink to a dir
+ # returns true)
+ next if (-l "$to/$sub");
+ # Only auto-nuke our specially formed rysnc directories
+ next unless $sub =~ /^sync.[\d\.]+$/;
+ # Nuke it if we got to here.
+ print "CLEAN: $to/$sub\n";
+ rmtree("$to/$sub") unless($noact);
+ }
+ exit 0;
+}
+
+# Make a new directory name
+my $newcurrent = "sync.$$." . time;
+die "$newcurrent already exists, what a horrible coincidence\n"
+ if (-e "$to/$newcurrent");
+
+# Lists of commands to execute that we will populate
+my @copy = ();
+my @setlive = ();
+my @verify = ();
+
+# If we bomb out, nuke our new directory.
+my @abort = ['rm', '-rf', "$to/$newcurrent"];
+
+# If they asked for verification, add a verify step.
+if ($o{'md5-verify'}) {
+# @verify = (['verify-md5sum', "$to/$newcurrent"]); FIXME
+}
+
+# if they asked for external verification, add that
+# note this invokes a shell
+if ($o{'external-verify'}) {
+ for my $vrfycmd (@{$o{'external-verify'}}) {
+ push @verify, ["sh", "-c", "$vrfycmd $to/$newcurrent" ];
+ }
+}
+
+# If we have both current and previous symlinks, we'll hardlink files
+# in current to the new directory, then rsync to the new directory,
+# then to set live we set current to the new current, move previous
+# to the previous current, and nuke the old previous.
+if (defined($current) and defined($previous)) {
+ @copy = (['cp', @$cpargs, "$to/$current", "$to/$newcurrent"],
+ [$rzsync, @rzsync_opts, "--from", $from, "--to", "$to/$newcurrent"]);
+ #replace
+ @setlive = (['ln', '-sfn', $newcurrent, "$to/current"],
+ ['ln', '-sfn', $current, "$to/previous"],
+ ['rm', '-rf', "$to/$previous"]);
+}
+
+# If we have only a current symlink, we'll hardlink files in current
+# to the new directory, then rsync to the new directory, then to set
+# live we set current to the new current, and make previous point
+# to the previous current.
+elsif (defined($current)) {
+ @copy = (['cp', @$cpargs, "$to/$current", "$to/$newcurrent"],
+ [$rzsync, @rzsync_opts, "--from", $from, "--to", "$to/$newcurrent"]);
+ # replace with call to recursive_zsync_fetch
+ @setlive = (['ln', '-sfn', $newcurrent, "$to/current"],
+ ['ln', '-sfn', $current, "$to/previous"]);
+}
+
+# If we have neither a current nor a previous, we'll create the new
+# directory, then rsync to it, and set live by setting current to
+# point to the new directory.
+else {
+ @copy = (['mkdir', '-p', $to],
+ [$rzsync, @rzsync_opts, "--from", $from, "--to", "$to/$newcurrent"]);
+ @setlive = (['ln', '-sfn', $newcurrent, "$to/current"]);
+}
+
+# If we were asked to not actually do anything, just print out
+# the commands we would have run, and exit.
+if ($noact) {
+ local $" = "' '"; # attempt at showing quoting of args
+ print "RUN: ['@$_']\n" for (@copy, @verify, @setlive);
+ exit 0;
+}
+else {
+ # Do the copy commands, then any verification that was requested.
+ # If we run into problems, run the abort commands and exit.
+ foreach my $cmd (@copy, @verify) {
+ print "RUN: [@$cmd]\n";
+ system(@$cmd);
+ my $out = $?;
+ if ($out) {
+ foreach my $cmd2 (@abort) {
+ print "ABORT: [@$cmd2]\n";
+ system(@$cmd2);
+ }
+ die "atomic rsync failed: @$cmd exited " . ($out>>8) . "\n";
+ }
+ }
+ # Do the commands needed to set the new copy live. If we run into
+ # problems, run the abort commands.
+ foreach my $cmd (@setlive) {
+ print "RUN: [@$cmd]\n";
+ system(@$cmd);
+ my $out = $? >> 8;
+ if ($out) {
+ warn "atomic rsync setlive failed: @$cmd exited " . ($out>>8) . "\n";
+ }
+ }
+}
+
86 azsync/azsync-builddir
@@ -0,0 +1,86 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+#!/usr/local/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long;
+use Chisel::Manifest;
+use File::Basename qw();
+
+
+# take a manifest and build a zsync for every file
+
+our %opt = (
+ 'h|help' => 'Print this message',
+ 'i|in=s' => 'input directory, default .',
+ 'd|directory=s' => 'output directory, default azsync.data (will be inside --in)',
+ 'wipe' => 'forcefully rm -rf the old azsync.data -d arg',
+ 'zsyncmake=s' => 'path to zsyncmake command',
+ 'emit_keys=s@' => 'specific tags to include, '.
+ 'default: $Chisel::Manifest::defaults{emit_keys}',
+ );
+
+die "RTFS, bad usage" unless GetOptions(\our %o, keys %opt);
+
+$o{i} ||= ".";
+$o{d} ||= "azsync.data";
+
+die "-d can't have slashes/nested dir" if $o{d} =~ m!/!;
+$o{emit_keys} = $Chisel::Manifest::defaults{emit_keys}
+ unless $o{emit_keys};
+
+# chdir into --in, everything is relative to it
+chdir($o{i}) or die "chdir $o{i}: $!\n";
+
+if( $o{wipe} ) {
+ system "rm", "-rf", $o{d};
+ die "can't wipe $o{d}\n" if($?);
+
+ system "rm", "-f", "azsync.manifest.json";
+ die "can't wipe azsync.manifest.json\n" if($?);
+}
+
+my $manifest = Chisel::Manifest->new({ emit_keys => $o{emit_keys} });
+my $output = $manifest->add_dir(".")->compute_manifest->to_json_lines();
+
+{
+ open my $fh, ">", "azsync.manifest.json" or die "can't write azsync.manifest.json: $!";
+ print $fh $output;
+ close $fh;
+}
+
+my $zsyncmake = $o{zsyncmake};
+$zsyncmake ||= "/bin/zsyncmake";
+
+mkdir $o{d} or die "can't mkdir $o{d}: $!";
+# only generate a .zsync for the first name listed in any set of links
+ #
+for my $mf (@{$manifest->{computed_manifest}}) {
+ my $first_name = $mf->first_name;
+ my $relative_path_to_file = $first_name;
+ $relative_path_to_file = File::Basename::dirname($relative_path_to_file);
+ $relative_path_to_file = "" if $relative_path_to_file eq ".";
+ $relative_path_to_file =~ s![^/]+!..!g;
+ $relative_path_to_file .= "/" if $relative_path_to_file ne "";
+ $relative_path_to_file = "../" . $relative_path_to_file ; # azsync.data dir
+ $relative_path_to_file .= "$first_name";
+
+ system "mkdir", "-p", File::Basename::dirname("$o{d}/$first_name");
+ next unless -f $first_name;
+
+ unlink "$o{d}/$first_name";
+ system $zsyncmake, $first_name, "-o", "$o{d}/$first_name", "-u", $relative_path_to_file;
+ die "can't $zsyncmake $first_name\n" if($?);
+ die "didn't $zsyncmake $first_name\n" if ! -f "$o{d}/$first_name";
+}
+
240 azsync/recursive-zsync
@@ -0,0 +1,240 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+#!/usr/local/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long;
+use File::Path;
+use Chisel::Manifest;
+use LWP::UserAgent;
+use File::Spec ();
+use Fcntl;
+
+my %opt = (
+ 'h|help' => 'Print this message',
+ 'from=s' => 'Url to sync from, eg http://host/dir/',
+ 'to=s' => 'Directory to sync to',
+ 'curl' => 'Use curl instead of LWP to fetch the Manifest.',
+ 'sslcert=s' => "Client certificate file (SSL)",
+ 'sslkey=s' => "Private key file name (SSL)",
+ 'interface=s' => "Attempt to use this interface name, IP address or host name as the source of http connections",
+ 'request-timeout=i' => "Timeout for each http request made",
+);
+
+my %o;
+die "bad options" unless GetOptions( \%o, keys %opt );
+
+die "need --from url\n" unless $o{from};
+die "need --to directory\n" unless $o{to};
+
+recursive_zsync_fetch($o{from}, $o{to});
+
+exit 0;
+
+# $o{from} is of the form http://host/path/
+# and we assume http://host/path/file_manifest.json exists
+# structure is http://host/path/ -
+# /.zsync.data/
+# /.zsync.data/
+
+sub recursive_zsync_fetch {
+ my ($base_url, $local_path) = @_;
+ -d $local_path or mkdir $local_path or die "mkdir $local_path: $!";
+ # make my local manifest (from $local_path)
+ my $local_manifest = new Chisel::Manifest;
+ chdir($local_path) or die "can't chdir to $local_path: $!\n";
+ $local_manifest->add_dir(".")->compute_manifest;
+
+ # load the remote manifest (from $base_url/azsync.manifest.json)
+ my $remote_manifest = new Chisel::Manifest;
+
+ if( $o{curl} ) {
+ # Curl options used:
+ # -q for ignore config file,
+ # -s for silent,
+ # -S for show errors,
+ # -f fail with exit code (22),
+ my @curlopt = ( "-q", "-s", "-S", "-f" );
+
+ # Respect --request-timeout for curl
+ if( $o{'request-timeout'} ) {
+ push @curlopt, "-m", $o{'request-timeout'};
+ }
+
+ # Respect --sslcert for curl
+ if( $o{'sslcert'} ) {
+ push @curlopt, "--cert", $o{'sslcert'};
+ }
+
+ # Respect --sslkey for curl
+ if( $o{'sslkey'} ) {
+ push @curlopt, "--key", $o{'sslkey'};
+ }
+
+ # Respect --interface for curl
+ if( $o{'interface'} ) {
+ push @curlopt, "--interface", $o{'interface'};
+ }
+
+ my $curlcmd = "curl " . join( " ", map { quotemeta $_ } @curlopt, "$base_url/azsync.manifest.json" );
+ my $response = qx[$curlcmd];
+ die "manifest fetch failed: $base_url\n" unless $? == 0;
+ $remote_manifest->load_manifest_data( $response );
+ } else {
+ my $ua = LWP::UserAgent->new( timeout => 10 );
+
+ # Respect --request-timeout for LWP
+ if( $o{'request-timeout'} ) {
+ $ua->timeout( $o{'request-timeout'} );
+ }
+
+ # Don't respect --sslcert or --sslkey for LWP
+ if( $o{'sslcert'} || $o{'sslkey'} ) {
+ warn "WARNING: Options --sslcert and --sslkey are ignored when using LWP.\n";
+ }
+
+ # Respect --interface for LWP
+ if( $o{'interface'} ) {
+ $ua->local_address( $o{'interface'} );
+ }
+
+ my $response = $ua->get( "$base_url/azsync.manifest.json" );
+ die "manifest fetch failed: $base_url\n" unless $response->is_success;
+ $remote_manifest->load_manifest_data( $response->decoded_content );
+ }
+
+ # first prune any extras, we need to do this in case a directory changed into a file or something
+ clean_extras($local_path,$remote_manifest);
+ # now iterate the manifest, and
+ # * create any directories
+ # * zsync once for every type=file
+ for my $remote_manifest_entry (@{$remote_manifest->{computed_manifest}}) {
+
+ # names that this file is known by
+ my ($first_name, @other_names) = $remote_manifest_entry->names;
+ my $type = $remote_manifest_entry->{data}{type};
+ # if there is an identical entry in the local_manifest, we can skip pulling this entry
+ next if $local_manifest->contains_entry($remote_manifest_entry, [ 'name', 'md5', keys %{$remote_manifest_entry->{data}} ]);
+ next if $first_name eq "./azsync.manifest.json"; # needed?
+ next unless $type eq 'file' || $type eq 'link';
+ # create paths for this file to go into
+ for my $n ($first_name, @other_names) {
+ my $dir_name = File::Basename::dirname($n);
+ if( $dir_name ne '.' ) {
+ die "insecure pathspec in manifest: $n\n" unless is_safe_path($local_path, $dir_name);
+ system_or_die( "mkdir", "-p", "$local_path/$dir_name" ) if ! -d "$local_path/$dir_name";
+ }
+ }
+
+ if( $type eq 'file' ) {
+ # Zsync options used:
+ # -q for quiet
+ my @zsync_opts = ( "-q" );
+
+ # Respect --request-timeout for zsync
+ if( $o{'request-timeout'} ) {
+ push @zsync_opts, "-T", $o{'request-timeout'};
+ }
+
+ # Respect --sslcert for zsync
+ if( $o{'sslcert'} ) {
+ push @zsync_opts, "-R", $o{'sslcert'};
+ }
+
+ # Respect --sslkey for zsync
+ if( $o{'sslkey'} ) {
+ push @zsync_opts, "-S", $o{'sslkey'};
+ }
+
+ # Respect --interface for zsync
+ if( $o{'interface'} ) {
+ push @zsync_opts, "-I", $o{'interface'};
+ }
+
+ system_or_die( "zsync", @zsync_opts, "-o", "$local_path/$first_name","$base_url/azsync.data/$first_name" );
+
+ # * if there were other names, hard link them
+ for my $n (@other_names) {
+ unlink "$local_path/$n";
+ link "$local_path/$first_name", "$local_path/$n" or
+ die "failed to hardlink $local_path/$first_name to $local_path/$n: $!\n";
+ }
+ } elsif( $type eq 'link' ) {
+ my $target = $remote_manifest_entry->{data}{link};
+ die "symlinks may only have one name, manifest lists multiple ($first_name @other_names)\n" if @other_names;
+
+ unlink "$local_path/$first_name";
+ symlink $target, "$local_path/$first_name" or
+ die "failed to make symlink $local_path/$first_name -> $target: $!\n";
+ }
+ }
+ # clean extras again, sometimes zsync leaves little presents behind
+ clean_extras($local_path,$remote_manifest);
+ # chmod, chown, etc
+ $remote_manifest->enforce();
+ # make sure we did everything properly
+ $remote_manifest->validate();
+}
+
+# given a local path ($local_path) and Chisel::Manifest ($manifest), remove any extra
+# files and directories which are not in the manifest
+sub clean_extras {
+ my ($local_path,$manifest) = @_;
+
+ my @extra = ( $manifest->extra_files({path => $local_path}), $manifest->extra_dirs({path => $local_path}) );
+ for my $f (@extra) {
+ next if -d $f && ! -l $f;
+ unlink $f or die "can't unlink $f";
+ }
+ for my $f (reverse sort {length $a <=> length $b} @extra) {
+ next unless -d $f && ! -l $f;
+ rmdir $f or die "can't rmdir $f";
+ }
+
+ # check for extras again to make sure we got them all
+ @extra = ( $manifest->extra_files({path => $local_path}), $manifest->extra_dirs({path => $local_path}) );
+ die "can't unlink some extras: @extra" if @extra;
+}
+
+# figure out if some $inner_path escapes $outer_path
+# $inner_path does not need to fully exist, since it will be mkdir -p'd later
+# it just can't have symlinks, ..'s, etc
+sub is_safe_path {
+ my ($outer_path, $inner_path) = @_;
+ # $outer_path doesn't exist, wtf?
+ return 0 unless -d $outer_path;
+ # break $inner_path into components
+ my @d = File::Spec->splitdir($inner_path);
+ # there's no good reason to have these as filenames in the manifest and they could be bad
+ return 0 if grep { $_ eq '' || $_ eq '/' || $_ eq '.' || $_ eq '..' } @d;
+ # there's no good reason to use symlinks as part file paths in the manifest
+ # since it should be totally self contained
+ # build all the path prefixes and check if any of them are symlinks
+ for my $di (0..$#d) {
+ my $inner_path_prefix = File::Spec->catdir(@d[0..$di]);
+ return 0 if -l "$outer_path/$inner_path_prefix";
+ }
+ # looks ok
+ return 1;
+}
+
+sub system_or_die {
+ my @cmd = @_;
+
+ system(@cmd);
+ if($?) {
+ die "[FAILED] @cmd\n";
+ } else {
+ return $?;
+ }
+}
23 builder/bin/doozer
@@ -0,0 +1,23 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+use FindBin ();
+
+die unless @ARGV > 0;
+my $bin = $FindBin::Bin;
+my $subcommand = shift @ARGV;
+$bin =~ s!bin$!libexec/doozer/doozer-$subcommand! or die;
+-x $bin or die "doozer-$subcommand: not a doozer subcommand\n";
+exec( $bin, @ARGV );
+die;
45 builder/conf/builder.conf
@@ -0,0 +1,45 @@
+#!/usr/bin/perl -w
+
+use strict;
+use YAML::XS;
+
+my %ckeys = (
+ # setting # description
+ 'application' => 'application name for metrics',
+
+ 'chunksize' => 'build for top X hosts in chunks, instead of monolithically building for all hosts',
+ 'generate_threads' => 'generation concurrency level',
+ 'pack_threads' => 'packing concurrency level',
+
+ 'gnupghome' => 'like GNUPGHOME, see gpg(1)',
+ 'build_interval' => 'amount of time to wait in between runs of doozer-build',
+ 'checkout_interval' => 'amount of time to wait in between runs of doozer-checkout',
+ 'gc_interval' => 'amount of time to wait in between runs of doozer-gc',
+ 'walrus_interval' => 'amount of time to wait in between runs of doozer-build-walrus',
+
+ 'log4perl_level' => 'Log::Log4perl logging level (like DEBUG or INFO)',
+ 'log4perl_pattern' => 'Log::Log4perl logging pattern (for PatternLayout)',
+
+ 'sanity_port' => 'sanity checker service port',
+ 'sanity_server' => 'sanity checker service hostname or ip',
+
+ 'ssh_identity' => 'ssh identity for fetching from data sources (private key)',
+ 'ssh_known_hosts' => 'ssh known_hosts file for fetching from data sources',
+ 'ssh_user' => 'ssh username for fetching from data sources (e.g. "chiseldata")',
+ 'svn_url' => 'svn configuration url',
+
+ 'var' => 'location of our "home" directory',
+
+ 'cluster' => 'cluster members, semicolon-separated (like foo;bar;baz)',
+ 'cluster_redundancy' => 'number of cluster members to serve each host from',
+ 'cluster_yca_appid' => 'yca appid for crosstalk on the /v2 api',
+ 'cluster_vip' => 'VIP to use when connecting to serving cluster',
+ 'zookeeper_connect' => 'zookeeper connection string (like localhost:2181)',
+ 'zookeeper_dir' => 'zookeeper data directory',
+);
+
+
+print <<EOT;
+
+$yaml
+EOT
393 builder/lib/Chisle/Builder/Engine.pm
@@ -0,0 +1,393 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+package Chisel::Builder::Engine;
+
+use strict;
+
+use Digest::MD5 ();
+use Fcntl qw/:DEFAULT :flock/;
+use Hash::Util ();
+use IO::Socket::INET;
+use Log::Log4perl qw/:easy/;
+use Net::ZooKeeper;
+use Sys::Hostname ();
+use YAML::XS ();
+use Chisel::Builder::Engine::Actuate;
+use Chisel::Builder::Engine::Checkout;
+use Chisel::Builder::Engine::Generator;
+use Chisel::Builder::Engine::Packer;
+use Chisel::Builder::Engine::Walrus;
+use Chisel::Builder::ZooKeeper::Leader;
+use Chisel::Builder::ZooKeeper::Worker;
+use Chisel::Metrics;
+use Regexp::Chisel qw/:all/;
+
+sub new {
+ my ( $class, %userconfig ) = @_;
+
+ my $self = {
+ configfile => ( delete $userconfig{'configfile'} || "/conf/builder.conf" ),
+ application => undef, # application to use for metrics
+ metrics => undef, # undef means fill it when asked
+ config => undef, # undef means load from 'configfile'
+ is_setup => undef, # have we had ->setup called yet?
+ keydb_init => undef, # have we called ycrKeyDbInit yet?
+ userconfig => undef, # overlay on top of 'config'
+ };
+
+ $self->{'userconfig'} = \%userconfig;
+
+ bless $self, $class;
+ Hash::Util::lock_keys( %$self );
+ return $self;
+}
+
+# basic setup:
+# - make sure we're not running as root
+# - read config file
+# - init Log4perl
+sub setup {
+ my ( $self, %args ) = @_;
+
+ # return if we already are setup
+ return if $self->is_setup;
+
+ if(!$args{'root_ok'} && $< == 0) {
+ die "please do not run this as root\n";
+ }
+
+ my $l4p_level = $self->config( "log4perl_level" );
+ my $l4p_pattern = $self->config( "log4perl_pattern" );
+ my $l4p_config = <<EOT;
+log4perl.rootLogger = $l4p_level, stderr
+log4perl.logger.Group.Roles = WARN, stderr
+log4perl.logger.Net.SSH = WARN, stderr
+log4perl.appender.stderr = Log::Log4perl::Appender::Screen
+log4perl.appender.stderr.layout = PatternLayout
+log4perl.appender.stderr.layout.ConversionPattern = $l4p_pattern
+log4perl.appender.stderr.stderr = 1
+EOT
+
+ Log::Log4perl->init( \$l4p_config );
+
+ # set is_setup so we know this all happened
+ $self->{is_setup} = 1;
+
+ # return $self for chaining
+ return $self;
+}
+
+# has ->setup been called yet?
+sub is_setup {
+ my ( $self ) = @_;
+ return $self->{is_setup};
+}
+
+# flock a particular file (the "keyword"), and return a handle to it
+sub lock {
+ my ( $self, $keyword, %args ) = @_;
+
+ my $lockfd;
+ my $lockdir = $self->config( "var" ) . "/lock";
+
+ my $lockp = "$lockdir/$keyword";
+ sysopen $lockfd, $lockp, O_WRONLY | O_CREAT
+ or LOGCROAK "open $lockp: $!";
+
+ # allow block => 1 to wait for ability to lock
+ my $flags = $args{'block'} ? LOCK_EX : LOCK_EX | LOCK_NB;
+
+ flock $lockfd, $flags
+ or LOGCROAK "lock $lockp: $!";
+ return $lockfd;
+}
+
+# return handle to metrics object
+# there is only ONE of these
+sub metrics {
+ my ( $self ) = @_;
+
+ $self->{metrics} ||= Chisel::Metrics->new(
+ application => $self->config( 'application' ),
+
+ );
+
+ return $self->{metrics};
+}
+
+# Stubbed for now. CMDB::Client does not exist.
+# return handle to cmdb client object
+# there can be MORE THAN ONE of these
+sub cmbddb {
+ my ( $self, %args ) = @_;
+
+ my $cmdb = CMDB::Client->new(
+ host => $self->config( "cmdb_url" ),
+ user => $self->config( "cmdb_user" ),
+ pass => $self->config( "cmdb_pass" ),
+ %args,
+ );
+
+ return $cmdb;
+}
+
+# Stubbed for now. Group::Client does not exist
+# return handle to roles client object
+# there can be MORE THAN ONE of these
+sub roles {
+ my ( $self, %args ) = @_;
+
+ my $roles = Group::Client->new(
+ baseuri => $self->config( "group_url" ) || undef, # turn '' into undef
+ %args,
+ );
+
+ return $roles;
+}
+
+# return an Actuate engine
+# there can be MORE THAN ONE of these so save a reference if you need it
+sub new_actuate {
+ my ( $self, %args ) = @_;
+
+ # default location for various important directories
+ my $var = $self->config( "var" );
+ $args{'indir'} ||= "$var/indir";
+
+ # fill in some parameters from our configuration, if they aren't overridden by the caller
+ $args{$_} = $self->config( $_ ) for grep { !exists $args{$_} } qw/ svn_url
+ ssh_user
+ ssh_identity
+ ssh_known_hosts /;
+
+ # pass in our shared metrics object
+ $args{'metrics_obj'} ||= $self->metrics;
+
+ return Chisel::Builder::Engine::Actuate->new( %args );
+}
+
+# return a Checkout engine
+# there can be MORE THAN ONE of these so save a reference if you need it
+sub new_checkout {
+ my ( $self, %args ) = @_;
+
+ # default location for various important directories
+ my $var = $self->config( "var" );
+ $args{'transformdir'} ||= "$var/indir/transforms";
+ $args{'tagdir'} ||= "$var/indir/tags";
+ $args{'scriptdir'} ||= "$var/modules";
+
+ if( !exists $args{'rawobj'} ) {
+ # delete $args{'rawdir'} since it's not actually going to be passed to a Checkout object
+ # we're deleting it inside this conditional so 'rawdir' and 'rawobj' conflict with each other (as they should)
+ my $rawdir = delete $args{'rawdir'} || "$var/indir/raw";
+
+ # set up the dynamic raw filesystem
+ my $r = Chisel::Builder::Raw->new;
+ my $r_fs = Chisel::Builder::Raw::Filesystem->new( rawdir => $rawdir, );
+ my $r_roles = Chisel::Builder::Raw::Roles->new( c => $self->roles );
+ my $r_usergroup = Chisel::Builder::Raw::UserGroup->new( c => $self->cmdb, );
+
+ my $r_hostlist = Chisel::Builder::Raw::HostList->new(
+ group_client => $self->roles,
+ range_tag => $self->config( 'range_tag' ),
+ maxchange => $self->config( 'range_maxchange' ),
+ cache_file => $self->config( "var" ) . "/cache/hostlist-sqlite",
+ );
+
+ # for reading files directly out of svn
+ $r->mount( plugin => $r_fs, mountpoint => "/" );
+
+ # XXX sort of a hack to treat module scripts as raw files
+ # XXX and to allow modules to bundle arbitrary files
+ foreach my $module ( glob "$args{scriptdir}/*" ) {
+ $module =~ s{^.*/([^/]+)$}{$1};
+ next unless $module;
+
+ my $module_script_fs =
+ Chisel::Builder::Raw::Filesystem->new( rawdir => "$args{scriptdir}/$module/scripts" );
+ my $module_file_fs =
+ Chisel::Builder::Raw::Filesystem->new( rawdir => "$args{scriptdir}/$module/files" );
+
+ $r->mount( plugin => $module_script_fs, mountpoint => "/modules/$module" );
+ $r->mount( plugin => $module_file_fs, mountpoint => "/${module}.bundle" );
+ }
+
+ # virtual mountpoints for roles and cmdb usergroups, used for invokefor
+ $r->mount( plugin => $r_roles, mountpoint => "/group_role" );
+ $r->mount( plugin => $r_usergroup, mountpoint => "/cmdb_usergroup" );
+
+ # virtual mountpoints for magically imported raw files
+ $r->mount( plugin => $r_hostlist, mountpoint => "/internal/hostlist" );
+
+ $args{'rawobj'} = $r;
+ }
+
+ # pass in our shared metrics object
+ $args{'metrics_obj'} ||= $self->metrics;
+
+ return Chisel::Builder::Engine::Checkout->new( %args );
+}
+
+sub new_walrus {
+ my ( $self, %args ) = @_;
+
+ if( !exists $args{'transforms'} ) {
+ LOGCROAK "please pass in transforms";
+ }
+
+ if( !exists $args{'tags'} ) {
+ LOGCROAK "please pass in tags";
+ }
+
+ if( !exists $args{'groupobj'} ) {
+ # this is what the walrus will use to assign transforms and tags to hosts
+ my $g = Chisel::Builder::Group->new;
+ my $g_host = Chisel::Builder::Group::Host->new;
+ my $g_roles = Chisel::Builder::Group::Roles->new(
+ c => $self->roles,
+ threads => $self->config( "roles_threads" ),
+ cache_file => $self->config( "var" ) . "/cache/roles-sqlite",
+ turbo => $self->config( "roles_turbo" ),
+ );
+ my $g_cmdb_node = Chisel::Builder::Group::cmdbNode->new(
+ c => $self->cmdb,
+ cache_file => ( $self->config( "var" ) . "/cache/cmdb-sqlite" ),
+ turbo => $self->config( "cmdb_turbo" ),
+ );
+
+ $g->register( plugin => $g_host );
+ $g->register( plugin => $g_roles );
+ $g->register( plugin => $g_cmdb_node );
+ $g->register( plugin => $g_cmdb_nodegroup );
+
+ $args{'groupobj'} = $g;
+ }
+
+ if( !exists $args{'require_group'} ) {
+
+ # require_group can prevent inconsistencies in roles api calls from causing problems
+ WARN "new_walrus: Consider using 'require_group' as a safety measure";
+ }
+
+ # fill in some parameters from our configuration, if they aren't overridden by the caller
+ $args{$_} = $self->config( $_ ) for grep { !exists $args{$_} } qw/ require_tag /;
+
+ # pass in our shared metrics object
+ $args{'metrics_obj'} ||= $self->metrics;
+
+ return Chisel::Builder::Engine::Walrus->new( %args );
+}
+
+sub new_generator {
+ my ( $self, %args ) = @_;
+
+ # default location for various important directories
+ $args{'workspace'} ||= $self->config( "var" ) . "/ws";
+
+ return Chisel::Builder::Engine::Generator->new( %args );
+}
+
+sub new_packer {
+ my ( $self, %args ) = @_;
+
+ # default location for various important directories
+ $args{'workspace'} ||= $self->config( "var" ) . "/ws";
+ $args{'gnupghome'} ||= $self->config( "gnupghome" );
+
+ # 'sanity_socket' will be established unless passed in (it can be passed in undef to omit this)
+ if( !exists $args{'sanity_socket'} ) {
+ # delete $args{'sanity_server'} and $args{'sanity_port'} since it's not actually going to be passed to a Generator object
+ my $host = delete $args{'sanity_server'} || $self->config( "sanity_server" ) || "localhost";
+ my $port = delete $args{'sanity_port'} || $self->config( "sanity_port" ) || 10001;
+
+ $args{'sanity_socket'} = IO::Socket::INET->new(
+ PeerAddr => $host,
+ PeerPort => $port,
+ Proto => 'tcp',
+ ) or die "Can't create a socket to Sanity server ($host:$port): $!\n";
+ }
+
+ return Chisel::Builder::Engine::Packer->new( %args );
+}
+
+sub new_workspace {
+ my ( $self, %args ) = @_;
+ return Chisel::Workspace->new( dir => $self->config( "var" ) . "/ws", %args );
+}
+
+# return handle to ZooKeeper leader object
+sub new_zookeeper_leader {
+ my ( $self ) = @_;
+
+ return Chisel::Builder::ZooKeeper::Leader->new(
+ connect => $self->config("zookeeper_connect"),
+ redundancy => $self->config("cluster_redundancy"),
+ cluster => [ split ';', $self->config("cluster") ],
+ );
+}
+
+# return handle to ZooKeeper worker object
+# there is only ONE of these
+sub new_zookeeper_worker {
+ my ( $self, $worker ) = @_;
+
+ if( !$worker ) {
+ # Default worker name is our hostname
+ $worker = Sys::Hostname::hostname();
+ }
+
+ return Chisel::Builder::ZooKeeper::Worker->new( connect => $self->config("zookeeper_connect"), worker => $worker );
+}
+
+# read a key out of our config file
+sub config {
+ my ( $self, $key, $opts ) = @_;
+
+ if( !defined $self->{config} ) {
+ # this dies on failure, that's ok
+ my $config = YAML::XS::LoadFile( $self->{configfile} );
+
+ # overlay the userconfig
+ %$config = ( %$config, %{ $self->{'userconfig'} } );
+
+ # save our config
+ $self->{config} = $config;
+ }
+
+ # config is loaded, just return whatever was asked for
+ if( exists $self->{config}{$key} ) {
+
+ # return the value, whether or not it came from keydb
+
+ return $self->{config}{$key};
+ } else {
+ # err, what
+ # settings should always be available in the conf file even if they're undef
+ # this probably means whatever code called ->config has a typo in it, let's die as a safeguard
+
+ # need to use die because log4perl may not be inited yet
+ die "setting '$key' does not seem to exist\n";
+ }
+}
+
+# Scrubs possibly sensitive strings from error messages.
+sub scrub_error {
+ my ( $self, $message ) = @_;
+
+ $message =~ s/at .+ line \d+.*//sg; # remove stack traces
+ $message =~ s/\$\d\$[a-zA-Z0-9\.\/\$]{8,}/XXXXXXXXXXXX/sg; # just in case
+ $message =~ s/\s+\z//g; # remove trailing whitespace
+
+ return $message;
+}
+
+1;
124 builder/lib/Chisle/Builder/Engine/Actuate.pm
@@ -0,0 +1,124 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+package Chisel::Builder::Engine::Actuate;
+
+use 5.008;
+
+use strict;
+use warnings;
+use Algorithm::Diff;
+use Carp;
+use Data::Dumper;
+use Hash::Util ();
+use SVN::Client;
+use Socket;
+
+sub new {
+ my ( $class, %options ) = @_;
+
+ my $self = {
+ %options
+ };
+
+ bless $self, $class;
+ Hash::Util::lock_keys( %$self );
+
+ $self->_setenv;
+
+ return $self;
+}
+
+sub logger {
+ Log::Log4perl->get_logger(__PACKAGE__);
+}
+
+sub svn_checkout {
+ my ( $self ) = @_;
+
+ my $checkout_rev; # the revision we got back from svn co
+
+ my $rev = 'HEAD'; # don't ask for any particular revision
+ my $recurs = 1; # get subdirectories
+
+ # define where our configuration repository is
+ my $source = $self->{svn_url};
+
+ # construct an svn client object
+ $self->logger->info( "Begin checkout of $source to $self->{indir}" );
+ my $ctx = new SVN::Client( config => SVN::Core::config_get_config( "/etc/subversion" ) );
+
+ # svn revert our working copy of the repository
+ if ( -d "$self->{indir}" && -f "$self->{indir}/.svn/entries" ) {
+ $self->logger->debug( "sanitizing $self->{indir} : step 1 => svn revert" );
+ $ctx->revert( $self->{indir} , 1 );
+
+ # sanitize the working copy of any local changes
+ $self->logger->debug( "sanitizing $self->{indir} : step 2 => clear unmanaged files" );
+ my $status_handler = sub {
+ my ( $path, $status ) = @_;
+ if ( $status->text_status() == $SVN::Wc::Status::unversioned ) {
+ $self->logger->debug( "unlinking unversioned node $path found in working directory" );
+ if ( -f $path ) {
+ unlink( $path )
+ or $self->logger->debug( "unable to unlink unversioned file $path: $!" );
+ } elsif ( -d $path && $path =~ /^\Q$self->{indir}\E\//) {
+ system( "rm", "-rf", $path ) == 0
+ or $self->logger->debug( "unable to remove unversioned directory $path: $!" );
+ }
+ }
+ };
+ $ctx->status ( $self->{indir},
+ $rev,
+ $status_handler,
+ $recurs, 0, 0, 0,
+ );
+ }
+
+ # check the configuration out from the repository
+ $checkout_rev = $ctx->checkout ( $source,
+ $self->{indir},
+ $rev,
+ $recurs,
+ );
+ $self->logger->info( "Check out of revision $checkout_rev succeeded." );
+
+ return $checkout_rev;
+}
+
+sub _setenv {
+ my $self = shift;
+ $ENV{'SVN_SSH'} = "ssh -o UserKnownHostsFile=$self->{ssh_known_hosts} -o BatchMode=yes -i $self->{ssh_identity}";
+}
+
+sub indir {
+ my ( $self ) = @_;
+ return $self->{indir};
+}
+
+sub libexec {
+ my ( $self, @cmdline ) = @_;
+
+ $self->logger->info( "starting command $cmdline[0]" );
+ system(@cmdline) == 0
+ or croak "$cmdline[0] failed with code $?:$!";
+ $self->logger->info( "external command $cmdline[0] completed successfully" );
+ return $cmdline[0];
+}
+
+sub acquire_lock {
+ my $port = 11533;
+ my $proto = getprotobyname('tcp');
+ socket(Server, PF_INET, SOCK_STREAM, $proto) or return undef;
+ bind(Server, sockaddr_in($port, INADDR_ANY)) or return undef;
+ 1;
+}
+
+1;
306 builder/lib/Chisle/Builder/Engine/Checkout.pm
@@ -0,0 +1,306 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+package Chisel::Builder::Engine::Checkout;
+
+# this package is meant to read information out of a svn repository, as well as any needed "virtual" raw files like usergroups
+#
+# inputs: transformdir + rawdir + scriptdir + cmdb/roles credentials
+# outputs: transform objects, tag objects, all necessary raw files
+
+use strict;
+use warnings;
+use Chisel::Builder::Raw;
+use Chisel::Tag;
+use Chisel::Metrics;
+use Chisel::Transform;
+use Regexp::Chisel qw/ :all /;
+use Carp;
+use Log::Log4perl qw/:easy/;
+use Hash::Util ();
+use YAML::XS ();
+
+sub new {
+ my ( $class, %rest ) = @_;
+
+ my $defaults = {
+ # inputs
+ tagdir => '', # directory that tags live in
+ transformdir => '', # directory that transforms live in
+ scriptdir => '', # directory that modules live in
+
+ # obj to read raw files out of
+ rawobj => undef, # normally passed in by Engine
+
+ # set of ex-raw files from the last round
+ # used to provide context to the rawobj, for caching and sanity checking
+ ex_raws => [],
+
+ # Chisel::Metrics object for storing metrics
+ metrics_obj => undef,
+ };
+
+ my $self = { %$defaults, %rest };
+
+ # create objects that we need but weren't given to us
+ if( !$self->{metrics_obj} ) {
+ my ( $pkg, $file, undef ) = caller();
+ TRACE "We weren't given a metrics object by package $pkg in file $file, creating dummy";
+ $self->{metrics_obj} = Chisel::Metrics->new;
+ }
+
+ if( keys %$self > keys %$defaults ) {
+ LOGDIE "Too many parameters, expected only " . join ", ", keys %$defaults;
+ }
+
+ # add internals
+ %$self = (
+ %$self,
+
+ # internals and caches
+ modules => undef, # $modules{ module name } => its module.conf
+ transforms => undef, # array of Chisel::Transform objs
+ tags => undef, # array of Chisel::Tag objs
+ );
+
+ bless $self, $class;
+ Hash::Util::lock_keys(%$self);
+ return $self;
+}
+
+# either:
+# - return contents of a raw file
+# - OR return array of ALL interesting raw files
+sub raw {
+ my ( $self, $ufile ) = @_;
+
+ # lookup table of ex-raw-files (context for readraw later)
+ my %ex_raw_lookup = map { $_->name => $_ } @{$self->{ex_raws}};
+
+ if( @_ > 1 ) {
+ # caller wanted one raw file object
+ my $raw_obj = $self->{rawobj}->readraw($ufile, context => $ex_raw_lookup{$ufile});
+ if( defined $raw_obj && defined $raw_obj->data ) {
+ return $raw_obj;
+ } else {
+ die "file does not exist: $ufile";
+ }
+ } else {
+ # fetch all raw file objects we care about (based on our transforms)
+ my %raw_needed
+ = map { $_ => 1 } # remove duplicates by assigning to a hash
+ map { $_->raw_needed() } # all raw files needed by each transform
+ grep { $_->is_good() } # skip transforms that are unloadable
+ $self->transforms;
+
+ # now we're ready to fetch everything
+ my @raws;
+ foreach my $raw_name ( sort keys %raw_needed ) {
+ my $raw_obj = $self->{rawobj}->readraw( $raw_name, context => $ex_raw_lookup{$raw_name} );
+ if( defined $raw_obj ) {
+ push @raws, $raw_obj;
+ } else {
+ WARN "Skipped nonexistent raw file $raw_name";
+ }
+ }
+
+ return @raws;
+ }
+}
+
+# returns our array of Tag objects, filling $self->{tags} if necessary
+sub tags {
+ my ( $self, %args ) = @_;
+
+ if( ! defined $self->{tags} ) {
+ # tags are stored in here
+ my $tagdir = $self->{tagdir};
+
+ DEBUG "Scanning tags in $tagdir";
+
+ # we're going to read tags using a Chisel::Builder::Raw::Filesystem object
+ # they're pretty convenient and make sure no shenanigans are happening
+ my $tag_fs = Chisel::Builder::Raw::Filesystem->new( rawdir => $tagdir );
+
+ # we'll put them in here
+ my @tags;
+
+ foreach my $tag ( glob "$tagdir/*" ) {
+ # only look at regular files
+ next unless -f $tag;
+
+ # shorten the name, read it out of $tag_fs
+ if( $tag =~ m{/($RE_CHISEL_tag_key)$} ) {
+ my $shortname = $1;
+ my $name = $shortname eq 'GLOBAL' ? 'GLOBAL' : "cmdb_property/$shortname";
+ my $yaml = $tag_fs->fetch( $shortname );
+
+ if( ! defined $yaml ) {
+ LOGDIE "Tag [$name] cannot be read from $tagdir!";
+ }
+
+ if( my @dupes = grep { lc "$_" eq lc $name } @tags ) {
+ # checking for "duplicate" tags (ones that match case-insensitively)
+ LOGDIE "Duplicate tag keys: $name, " . join( ", ", @dupes );
+ }
+
+ # looks ok
+ push @tags, Chisel::Tag->new(
+ name => $name,
+ yaml => $yaml,
+ );
+ } else {
+ WARN "Ignoring tag file: $tag";
+ }
+ }
+
+ $self->metrics->set_metric( {}, 'n_tags', scalar @tags );
+ $self->{tags} = \@tags;
+ }
+
+ return @{ $self->{tags} };
+}
+
+
+# returns our array of Transform objects, filling $self->{transforms} if necessary
+sub transforms {
+ my ( $self, %args ) = @_;
+
+ if( ! defined $self->{transforms} ) { # fill $self->{transforms}
+ my $transformdir = $self->{transformdir};
+
+ my @t; # transform names
+ my @td; # subdirs in transforms/
+
+ DEBUG "Scanning transforms in $transformdir";
+
+ opendir my $dir, $transformdir
+ or confess( "Cannot open dir $transformdir" );
+
+ foreach my $f ( readdir $dir ) {
+ push @t, $f if $f =~ /^$RE_CHISEL_transform$/ && -f "$transformdir/$f";
+ push @td, $f if $f =~ /^$RE_CHISEL_transform_type$/ && -d "$transformdir/$f";
+ }
+
+ closedir $dir;
+
+ # read subdirs (only one level deep)
+ foreach my $d ( @td ) {
+ opendir my $dir, "$transformdir/$d"
+ or confess( "Cannot open dir $transformdir/$d" );
+
+ push @t,
+ map { "$d/$_" }
+ grep { /^$RE_CHISEL_transform_key$/ && -f "$transformdir/$d/$_" }
+ readdir $dir;
+ }
+
+ # we're going to read transforms using a Raw::Filesystem object
+ # they're pretty convenient and make sure no shenanigans are happening
+ my $transform_fs = Chisel::Builder::Raw::Filesystem->new( rawdir => $transformdir );
+
+ # read module.confs for all modules, we'll need to pass it to the transform objects
+ my %module_conf = map { $_ => $self->module( name => $_ ) } $self->modules;
+
+ # we have a list of all transforms in @t, let's create a %transforms hash
+ # it's ok for this to not include the contents, because this class deals only with one version of each
+ # $transforms{ lc transform name } = transform object
+ my %transforms;
+
+ foreach my $ti (@t) {
+ # read the yaml for this transform
+ my $ti_yaml = $transform_fs->fetch( $ti );
+
+ if( ! defined $ti_yaml ) {
+ LOGDIE "Transform [$ti] cannot be read from $transformdir!";
+ }
+
+ # create a transform object
+ my $transform = Chisel::Transform->new( name => $ti, yaml => $ti_yaml, module_conf => \%module_conf );
+
+ # add it to %transforms
+ confess "Duplicate transform key: $transform"
+ if exists $transforms{ lc $transform->name };
+ $transforms{ lc $transform->name } = $transform;
+ }
+
+ # add empty DEFAULT and DEFAULT_TAIL if there were no files for them
+ $transforms{'default'} = Chisel::Transform->new( name => 'DEFAULT', yaml => '', module_conf => \%module_conf )
+ if ! exists $transforms{'default'};
+
+ $transforms{'default_tail'} = Chisel::Transform->new( name => 'DEFAULT_TAIL', yaml => '', module_conf => \%module_conf )
+ if ! exists $transforms{'default_tail'};
+
+ # informational message
+ INFO "Read " . scalar( keys %transforms ) . " transforms";
+ $self->metrics->set_metric( {}, 'n_transforms', scalar keys %transforms );
+
+ # save the list
+ $self->{transforms} = [ values %transforms ];
+ }
+
+ # return the transform objects
+ return @{ $self->{transforms} };
+}
+
+# return a list of module names
+sub modules {
+ my ( $self, %args ) = @_;
+
+ return keys %{$self->{modules}} if defined $self->{modules};
+
+ my %module_conf;
+
+ my $scriptdir = $self->{scriptdir};
+ opendir my $dir, $scriptdir
+ or confess( "Cannot open modules dir: $scriptdir" );
+
+ my @scripts = grep { -d "$scriptdir/$_" && ! /^\./ } readdir $dir;
+ closedir $dir;
+
+ foreach my $s (@scripts) {
+ DEBUG "Reading module.conf for $s";
+
+ if( -f "$scriptdir/$s/module.conf" ) {
+ eval {
+ ( $module_conf{$s} ) = YAML::XS::LoadFile( "$scriptdir/$s/module.conf" );
+
+ 1;
+ } or do {
+ # YAML::XS::Load probably failed, die with a useful error
+ confess "Error loading module.conf for $s!\n$@";
+ };
+ } else {
+ $module_conf{$s} = {};
+ }
+ }
+
+ Hash::Util::lock_keys( %module_conf );
+ $self->{modules} = \%module_conf;
+ return keys %module_conf;
+}
+
+# return configuration for a module, or undef if the module does not exist
+sub module {
+ my ( $self, %args ) = @_;
+ defined( $args{$_} ) or confess( "$_ not given" )
+ for qw/name/;
+
+ $self->modules if ! defined $self->{modules};
+
+ return exists $self->{modules}{ $args{name} }
+ ? $self->{modules}{ $args{name} }
+ : undef;
+}
+
+# accessors
+sub metrics { return shift->{metrics_obj} }
+
+1;
248 builder/lib/Chisle/Builder/Engine/Generator.pm
@@ -0,0 +1,248 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+package Chisel::Builder::Engine::Generator;
+
+# inputs: desired generation targets (filename + transforms + raw files)
+# outputs: generated targets
+
+use strict;
+
+use Carp;
+use Digest::MD5 ( 'md5_hex' );
+use Encode ();
+use Hash::Util ();
+use JSON::XS ();
+use Log::Log4perl ( ':easy' );
+
+use Chisel::Transform;
+use Chisel::Workspace;
+use Regexp::Chisel ( ':all' );
+
+sub new {
+ my ( $class, %rest ) = @_;
+
+ my $defaults = {
+ workspace => '', # location of the workspace that we should update
+ };
+
+ my $self = { %$defaults, %rest };
+
+ if( keys %$self > keys %$defaults ) {
+ LOGDIE "Too many parameters, expected only " . join ", ", keys %$defaults;
+ }
+
+ # add internals
+ %$self = (
+ %$self,
+
+ # object for interacting with the repo in 'workspace'
+ workspace_obj => Chisel::Workspace->new( dir => $self->{workspace} ),
+
+ );
+
+ bless $self, $class;
+ Hash::Util::lock_keys( %$self );
+ return $self;
+}
+
+# %args: hash like
+# {
+# raw: [ obj1, obj2, ... ],
+# targets: [
+# { transforms => [ obj1, obj2, ... ], file => "files/motd/MAIN" },
+# ]
+# }
+# return: array that matches "targets" from input (index by index), like
+# [
+# { ok => 1, blob => "blob sha" },
+# { ok => 0, message => "error msg"},
+# ]
+sub generate {
+ my ( $self, %args ) = @_;
+
+ # 'targets' is an arrayref of things like:
+ # { transforms => [ ... ], file => "files/motd/MAIN" }
+
+ my @targets = @{ $args{'targets'} };
+
+ # 'raws' is an arrayref of RawFile objects:
+ # [ rf1, rf2, ... ]
+
+ my @raws = @{ $args{'raws'} };
+
+ # Create transform context out of @raws
+
+ my $transform_ctx = Chisel::Builder::Engine::Generator::Context->new( raws => \@raws );
+
+ # We will generate files, and push them into @result in the same order as @targets
+
+ my @result;
+
+ foreach my $target ( @targets ) {
+
+ # XXX TRACE "Target [$target->{key}] start";
+
+ my $contents;
+ my $r = eval {
+ $contents = $self->construct(
+ file => $target->{'file'},
+ transforms => $target->{'transforms'},
+ ctx => $transform_ctx,
+ );
+
+ 1;
+ };
+
+ # stop stopwatch, record only if successful
+ my $t_gen_target = ymonsb_sw_stop($sw_gen_target);
+
+ if( defined $r ) {
+ # success. $contents might be undef, that's ok (it means don't include the file)
+ # XXX TRACE "Target [$target->{key}] done OK";
+
+ my $blob = defined $contents ? $self->{'workspace_obj'}->store_blob( $contents ) : undef;
+
+ push @result, { ok => 1, blob => $blob, };
+ } else {
+ # failure
+ chomp( my $err = $@ );
+ # XXX TRACE "Target [$target->{key}] done FAILED [$err]";
+
+ push @result, { ok => 0, message => $err, };
+ }
+ }
+
+ # write out time to build all of @targets
+ my $t_generate = ymonsb_sw_stop($sw_generate);
+
+ return \@result;
+}
+
+# make 'file' from 'transforms'
+# returns generated contents
+# returns undef if the blob shouldn't exist for this 'file' / 'transforms' pair (usually due to unlink)
+# dies on error
+sub construct {
+ my ( $self, %args ) = @_;
+
+ defined( $args{$_} ) or confess( "$_ not given" ) for qw/ file transforms ctx /;
+
+ # XXX rawdir hack hack hack
+ if( $args{file} =~ m{^scripts/([^/]+(?<!\.asc))(?:\.asc|)$} ) {
+ # a script, chroot into its modules/blah/scripts directory
+ $args{ctx}->cd( "/modules/$1" );
+ } elsif( $args{file} =~ m{^files/} ) {
+ # a regular file, use the raw filesystem as-is
+ $args{ctx}->cd( "" );
+ } else {
+ confess "$args{file}: bad path";
+ }
+
+ DEBUG "Constructing file [$args{file}] from transforms ["
+ . join( ', ', map { $_->id } @{ $args{'transforms'} } ) . "]";
+
+ my $transform_model;
+
+ foreach my $t ( @{ $args{transforms} } ) {
+ next if !$t->does_transform( file => $args{file} );
+
+ # create $transform_model if it doesn't exist yet
+ if( !$transform_model ) {
+ my $transform_model_class = $t->model( file => $args{file} );
+ $transform_model = $transform_model_class->new( ctx => $args{ctx} );
+ }
+
+ my $ret = $t->transform( file => $args{file}, model => $transform_model );
+
+ # return 1 => keep going
+ # return 0 => stop and remove file
+ # undef => error
+
+ if( !defined $ret ) {
+ # fail if the transform thought something went wrong
+ confess( "Transforming $t for " . $args{file} . " failed" );
+ } elsif( $ret == 0 ) {
+ return undef;
+ } elsif( $ret != 1 ) {
+ # this transform is buggy
+ confess( "Transform $t for " . $args{file} . " returned nonsense" );
+ }
+ }
+
+ my $contents = $transform_model->text;
+
+ # convert to bytes if this is a character string
+ if( utf8::is_utf8( $contents ) ) {
+ $contents = Encode::encode_utf8( $contents );
+ }
+
+ # return contents
+ return $contents;
+}
+
+sub workspace { shift->{workspace_obj} }
+
+package Chisel::Builder::Engine::Generator::Context;
+
+use strict;
+
+use Carp;
+use Hash::Util ();
+use Log::Log4perl ( ':easy' );
+
+# Chisel::TransformModel classes (transform action implementations) needs a $ctx
+# argument that can provide it raw files. This is that context.
+
+sub new {
+ my ( $class, %rest ) = @_;
+
+ my $self = {
+ # hacky thing that stores usernames => passwd lines
+ map => undef,
+
+ # hash for looking up raw files
+ # XXX changed to case insensitive for now -- possibly forever, but would like it to become sensitive again
+ raw_lookup => { map { lc $_->name => $_->decode } @{ $rest{raws} } },
+
+ # XXX hacky thing for prepending a "working directory" onto ->readraw calls
+ raw_chdir => "",
+ };
+
+ bless $self, $class;
+ Hash::Util::lock_keys( %$self );
+ return $self;
+}
+
+sub cd {
+ my ( $self, $newchdir ) = @_;
+ $self->{raw_chdir} = $newchdir;
+}
+
+sub readraw {
+ my ( $self, %args ) = @_;
+
+ if( defined $args{file} ) {
+ # add raw_chdir and strip slashes (we end up allowing leading slashes, unlike Raw)
+ # XXX changed to case insensitive for now -- possibly forever, but would like it to become sensitive again
+ my $key = lc "$self->{raw_chdir}/$args{file}";
+ $key =~ s{^/+}{};
+
+ if( defined $self->{raw_lookup}{$key} ) {
+ return $self->{raw_lookup}{$key};
+ } else {
+ LOGDIE "file does not exist: $key";
+ }
+ } else {
+ LOGDIE "file not given\n";
+ }
+}
+
+1;
240 builder/lib/Chisle/Builder/Engine/Packer.pm
@@ -0,0 +1,240 @@
+######################################################################
+# Copyright (c) 2012, Yahoo! Inc. All rights reserved.
+#
+# This program is free software. You may copy or redistribute it under
+# the same terms as Perl itself. Please see the LICENSE.Artistic file
+# included with this project for the terms of the Artistic License
+# under which this project is licensed.
+######################################################################
+
+
+package Chisel::Builder::Engine::Packer;
+
+# inputs: proto-buckets containing hostnames + generated files
+# outputs: buckets with NODELIST, VERSION, REPO and sanity-checked and signed MANIFEST
+
+use strict;
+
+use Digest::MD5 qw/md5_hex/;
+use Hash::Util ();
+use Log::Log4perl qw/:easy/;
+use YAML::XS ();
+
+use Chisel::Bucket;
+use Chisel::Integrity;