lib/ec2nodefinder/src/awssign.erl @@ -21,32 +21,25 @@ And if you provide your S3 credentials and a bucket name in config.rb, `rake upl = Running on EC2 = -You need a JDK, the EC2 admin tools and, of course, OTP. For each of the nodes : - I recommend Kevin Smith's AMIs : http://weblog.hypotheticalabs.com/?p=383 They have everything bundled in. I expect this code to get into the AMI soon. + +On each of your instances : On Ubuntu : # install erlang -apt-get -y install openjdk-6-jre-headless unzip -wget http://s3.amazonaws.com/ec2-downloads/ec2-api-tools.zip -cd /usr/local/ -unzip ~/ec2-api-tools.zip -ln -s ec2-api-tools-1.3-30349/ ec2-api-tools - -#Upload your EC2 private key and certificate to your instance. -#Upload your release to your ec2 instance(s) (S3 is a good choice for hosting your release) +#Upload your release to your ec2 instance(s) sudo erl -boot start_sasl -noshell -eval 'release_handler:unpack_release("<release_name>").' -s init stop -erl -boot start_sasl -name server -kernel inet_dist_listen_min 21000 \ - inet_dist_listen_max 21005 -s inets -setcookie 'XXXXXXXXXXX' -s ec2nodefinder +erl -boot start_sasl -sname server -ec2nodefinder access '"<AWS_ACCESS>"' secret '"<AWS_SECRET>"' + -s inets -s crypto -setcookie 'XXXXXXXXXXX' -s ec2nodefinder -After a short while (mainly due to JVM startup time) : -(server@domU-12-31-39-02-65-83.compute-1.internal)3> nodes(). -['server@domU-12-31-39-02-6A-48.compute-1.internal'] -Nodes are discovered ! +(server@domU-XXXXXX.compute-1.internal)3> nodes(). +['server@domU-XXXXXXXX.compute-1.internal'] + +All nodes in the same security group are discovered ! = Final notes = @@ -59,8 +52,7 @@ That caused ec2nodefinder to fetch the public DNS name instead of the private on - You can quite easily reuse the build system. Add your OTP applications in lib/, they'll be built. -- the tar.gz are at least 3Mb, because the base OTP applications are bundled. This insures compatibility. - But I could find a way to not bundle them, I'd do that. + = Thanks = To charpi for the Rakefile README @@ -1,16 +1,14 @@ {application, ec2nodefinder, -[{description, "node discovery"}, +[{description, "automatic node discovery on EC2"}, {vsn, %VSN%}, {modules, %MODULES%}, {registered,[ec2nodefinder]}, -{applications, [kernel,stdlib, inets]}, +{applications, [kernel,stdlib, inets, crypto]}, {mod, {ec2nodefinder,[]}}, {start_phases, []}, {env, [ {ping_timeout_sec, 10}, - {private_key, "/tmp/pk-XXXXXXXXXXXXXXXXXX.pem"}, - {cert, "/tmp/cert-XXXXXXXXXXXXXXXXXX.pem"}, - {ec2_home, "/usr/local/ec2-api-tools"}, - {java_home, "/usr/lib/jvm/java-6-openjdk"} + {access, ""}, + {secret, ""} ]} ]}. lib/ec2nodefinder/src/ec2nodefinder.app.src @@ -36,17 +36,13 @@ start (_Type, _Args) -> _ -> first_security_group () end, { ok, PingTimeout } = application:get_env (ec2nodefinder, ping_timeout_sec), - { ok, PrivateKey } = application:get_env (ec2nodefinder, private_key), - { ok, Cert } = application:get_env (ec2nodefinder, cert), - { ok, Ec2Home } = application:get_env (ec2nodefinder, ec2_home), - { ok, JavaHome } = application:get_env (ec2nodefinder, java_home), - + { ok, Access } = application:get_env (ec2nodefinder, access), + { ok, Secret } = application:get_env (ec2nodefinder, secret), + ec2nodefindersup:start_link (Group, 1000 * PingTimeout, - PrivateKey, - Cert, - Ec2Home, - JavaHome). + Access, + Secret). %% @hidden lib/ec2nodefinder/src/ec2nodefinder.erl @@ -5,6 +5,7 @@ [{kernel, "%KERNEL_VSN%"}, {stdlib, "%STDLIB_VSN%"}, {sasl, "%SASL_VSN%"}, - {inets, "%SASL_VSN%"}, + {inets, "%INETS_VSN%"}, + {crypto, "%CRYPTO_VSN%"}, {ec2nodefinder, "%VSN%"}] }. \ No newline at end of file lib/ec2nodefinder/src/ec2nodefinder.rel.src @@ -4,7 +4,7 @@ -module (ec2nodefindersrv). -behaviour (gen_server). --export ([ start_link/6, discover/0 ]). +-export ([ start_link/4, discover/0 ]). -export ([ init/1, handle_call/3, handle_cast/2, @@ -14,26 +14,25 @@ -record (state, { group, ping_timeout, - private_key, - cert, - ec2_home, - java_home }). + access, + secret + }). +-define(APIVERSION, "2008-12-01"). +-define(ENDPOINT, "ec2.amazonaws.com"). %-=====================================================================- %- Public - %-=====================================================================- -start_link (Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome) +start_link (Group, PingTimeout, Access, Secret) when is_list (Group), is_integer (PingTimeout), - is_list (PrivateKey), - is_list (Cert), - is_list (Ec2Home), - is_list (JavaHome) -> + is_list (Access), + is_list (Secret) -> gen_server:start_link ({ local, ?MODULE }, ?MODULE, - [ Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome ], + [ Group, PingTimeout, Access, Secret ], []). discover () -> @@ -43,19 +42,14 @@ discover () -> %- gen_server callbacks - %-=====================================================================- -init ([ Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome ]) -> +init ([ Group, PingTimeout, Access, Secret ]) -> pong = net_adm:ping (node ()), % don't startup unless distributed - { ok, _ } = file:read_file_info (PrivateKey), - { ok, _ } = file:read_file_info (Cert), - process_flag (trap_exit, true), State = #state{ group = Group, ping_timeout = PingTimeout, - private_key = PrivateKey, - cert = Cert, - ec2_home = Ec2Home, - java_home = JavaHome }, + access = Access, + secret = Secret }, discover (State), { ok, State }. @@ -94,62 +88,22 @@ collect (Key, Timeout) -> end. discover (State) -> - Command = "env EC2_HOME='" ++ shell_escape (State#state.ec2_home) ++ "' " ++ - "JAVA_HOME='" ++ shell_escape (State#state.java_home) ++ "' " ++ - shell_escape (State#state.ec2_home) ++ "/bin/ec2-describe-instances " ++ - "-K '" ++ shell_escape (State#state.private_key) ++ "' " ++ - "-C '" ++ shell_escape (State#state.cert) ++ "' " ++ - "2>/dev/null", - Output = os:cmd (Command), - + Group = State#state.group, Timeout = State#state.ping_timeout, - + Access = State#state.access, + Secret = State#state.secret, [ { Node, collect (Key2, Timeout) } || { Node, Key2 } <- [ { Node, start_ping (Node, Timeout) } || { Host, { ok, NamesAndPorts } } <- [ { Host, collect (Key, Timeout) } || { Host, Key } <- [ { Host, start_names (Host, Timeout) } - || Host <- parse_host_list (Output, Group) ] ], + || Host <- awssign:describe_instances(Group, ?ENDPOINT, ?APIVERSION, Access, Secret) ] ], { Name, _ } <- NamesAndPorts, Node <- [ a(Name ++ "@" ++ Host) ] ] ]. -%RESERVATION r-817d94e8 038239462139 default -%INSTANCE i-72f4021b ami-81d93ce8 ec2-67-202-6-8.z-1.compute-1.amazonaws.com domU-12-31-36-00-0C-02.z-1.compute-1.internal shutting-down paul 1 m1.small 2007-12-10T22:26:49+0000 -%RESERVATION r-f0927a99 038239462139 default -%INSTANCE i-290dfb40 ami-81d93ce8 ec2-67-202-18-240.compute-1.amazonaws.com domU-12-31-38-00-38-E6.compute-1.internal running paul 0 m1.small 2007-12-15T00:35:20+0000 -%RESERVATION r-d9927ab0 038239462139 flass -%INSTANCE i-310dfb58 ami-81d93ce8 ec2-72-44-51-185.z-1.compute-1.amazonaws.com domU-12-31-36-00-31-03.z-1.compute-1.internal running paul 0 m1.small 2007-12-15T00:37:05+0000 - -parse_host_list (Output, Group) -> - parse_host_list - ([ string:tokens (Line, "\t") || Line <- string:tokens (Output, "\n") ], - Group, - false, - []). -parse_host_list ([], _, _, Acc) -> - Acc; -parse_host_list ([ [] | Rest ], Group, State, Acc) -> - parse_host_list (Rest, Group, State, Acc); -parse_host_list ([ Line | Rest ], Group, _, Acc) - when hd (Line) =:= "RESERVATION", - length (Line) >= 4 -> - parse_host_list (Rest, Group, hd (tl (tl (tl (Line)))) =:= Group, Acc); -parse_host_list ([ Line | Rest ], Group, true, Acc) - when hd (Line) =:= "INSTANCE", - length (Line) >= 4 -> - parse_host_list (Rest, Group, true, [ hd (tl (tl (tl (tl(Line))))) | Acc ]); -parse_host_list ([ _ | Rest ], Group, State, Acc) -> - parse_host_list (Rest, Group, State, Acc). - -shell_escape (String) -> shell_escape (String, []). - -shell_escape ([], Acc) -> lists:reverse (Acc); -shell_escape ([ $' | T ], Acc) -> shell_escape (T, [ $', $\\ | Acc ]); -shell_escape ([ $\\ | T ], Acc) -> shell_escape (T, [ $\\, $\\ | Acc ]); -shell_escape ([ H | T ], Acc) -> shell_escape (T, [ H | Acc ]). start_names (Host, Timeout) -> async (fun () -> net_adm:names (a(Host)) end, Timeout). lib/ec2nodefinder/src/ec2nodefindersrv.erl @@ -3,24 +3,24 @@ -module (ec2nodefindersup). -behaviour (supervisor). --export ([ start_link/6, init/1 ]). +-export ([ start_link/4, init/1 ]). %-=====================================================================- %- Public - %-=====================================================================- -start_link (Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome) -> +start_link (Group, PingTimeout, Access, Secret) -> supervisor:start_link (?MODULE, - [ Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome ]). + [ Group, PingTimeout, Access, Secret ]). -init ([ Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome ]) -> +init ([ Group, PingTimeout, Access, Secret ]) -> { ok, { { one_for_one, 3, 10 }, [ { ec2nodefindersrv, { ec2nodefindersrv, start_link, - [ Group, PingTimeout, PrivateKey, Cert, Ec2Home, JavaHome ] }, + [ Group, PingTimeout, Access, Secret ] }, permanent, 10000, worker, lib/ec2nodefinder/src/ec2nodefindersup.erl @@ -1,2 +1,2 @@ -{vsn,"0.0.9"}. -{release_name,"ec2"}. \ No newline at end of file +{vsn,"0.1.1"}. +{release_name,"martin1"}. \ No newline at end of file lib/ec2nodefinder/vsn.config 7ca4559130517af021286afa8e2c103b0928960a Eric Cestari ecestari@mac.com http://github.com/cstar/ec2nodefinder/commit/935c4451f95090de1bae565f6f9f56344de3cf7c 935c4451f95090de1bae565f6f9f56344de3cf7c 2009-02-12T05:33:54-08:00 2009-02-12T05:33:54-08:00 Implemented native query interface call to the ec2. Now much faster and with a lot less dependencies. Also removed the need for the cert and private key. 539ef2c3014c623db3e498f84ee6e0d7556b8383 Eric Cestari ecestari@mac.com