mysqld_multi

#!/usr/bin/perl

# Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; version 2
# of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
# MA 02110-1301, USA
#
# -- modifications for Pinterest usage:
# : Bumping version to 2.20
# : Modified to support config regeneration at start time
# : auto-preload of jemalloc

use Getopt::Long;
use POSIX qw(strftime getcwd);
use File::Path qw(mkpath);
use 5.010;

$|=1;
$VER="2.20";

# These are not overridable via my.cnf, but if you don't use jemalloc
# or you don't use the config builder, you can always comment them out.
# The idea behind min/max jemalloc lib version is to support old boxes
# with libjemalloc.so.1 and newer ones with libjemalloc.so.2.  It's
# also worth noting that this is ubuntu-specific; we aren't really going
# for cross-platform compatibility here.
#
$jemalloc_base_path   = "/usr/lib/x86_64-linux-gnu/libjemalloc.so.";
$cnf_builder_path     = "/usr/local/bin/mysql_utils/mysql_cnf_builder.py";
$max_jemalloc_version = 2;
$min_jemalloc_version = 1;

my @defaults_options;   #  Leading --no-defaults, --defaults-file, etc.

$opt_example       = 0;
$opt_help          = 0;
$opt_log           = undef();
$opt_mysqladmin    = "/usr/bin/mysqladmin";
$opt_mysqld        = "/usr/sbin/mysqld";
$opt_no_log        = 0;
$opt_tcp_ip        = 0;
$opt_version       = 0;
$opt_silent        = 0;
$opt_verbose       = 0;

my $my_print_defaults_exists= 1;
my $logdir= undef();

my ($mysqld, $mysqladmin, $groupids, $homedir, $my_progname);

$homedir = $ENV{HOME};
$my_progname = $0;
$my_progname =~ s/.*[\/]//;


if (defined($ENV{UMASK})) {
  my $UMASK = $ENV{UMASK};
  my $m;
  my $fmode = "0640";

  if(($UMASK =~ m/[^0246]/) || ($UMASK =~ m/^[^0]/) || (length($UMASK) != 4)) {
    printf("UMASK must be a 3-digit mode with an additional leading 0 to indicate octal.\n");
    printf("The first digit will be corrected to 6, the others may be 0, 2, 4, or 6.\n"); }
  else {
    $fmode= substr $UMASK, 2, 2;
    $fmode= "06${fmode}"; }

  if($fmode != $UMASK) {
    printf("UMASK corrected from $UMASK to $fmode ...\n"); }

  $fmode= oct($fmode);

  umask($fmode);
}


main();

####
#### main sub routine
####

sub main
{
  my $flag_exit= 0;

  if (!defined(my_which(my_print_defaults)))
  {
    # We can't throw out yet, since --version, --help, or --example may
    # have been given
    print "WARNING: my_print_defaults command not found.\n";
    print "Please make sure you have this command available and\n";
    print "in your path. The command is available from the latest\n";
    print "MySQL distribution.\n";
    $my_print_defaults_exists= 0;
  }

  # Remove leading defaults options from @ARGV
  while (@ARGV > 0)
  {
    last unless $ARGV[0] =~
      /^--(?:no-defaults$|(?:defaults-file|defaults-extra-file)=)/;
    push @defaults_options, (shift @ARGV);
  }

  foreach (@defaults_options)
  {
    $_ = quote_shell_word($_);
  }

  # Add [mysqld_multi] options to front of @ARGV, ready for GetOptions()
  unshift @ARGV, defaults_for_group('mysqld_multi');

  # We've already handled --no-defaults, --defaults-file, etc.
  if (!GetOptions("help", "example", "version", "mysqld=s", "mysqladmin=s",
                  "log=s", "no-log",
                  "tcp-ip",  "silent", "verbose"))
  {
    $flag_exit= 1;
  }
  usage() if ($opt_help);

  if ($opt_verbose && $opt_silent)
  {
    print "Both --verbose and --silent have been given. Some of the warnings ";
    print "will be disabled\nand some will be enabled.\n\n";
  }

  init_log() if (!defined($opt_log));
  $groupids = $ARGV[1];
  if ($opt_version)
  {
    print "$my_progname version $VER by Jani Tolonen\n";
    exit(0);
  }
  example() if ($opt_example);
  if ($flag_exit)
  {
    print "Error with an option, see $my_progname --help for more info.\n";
    exit(1);
  }
  if (!defined(my_which(my_print_defaults)))
  {
    print "ABORT: Can't find command 'my_print_defaults'.\n";
    print "This command is available from the latest MySQL\n";
    print "distribution. Please make sure you have the command\n";
    print "in your PATH.\n";
    exit(1);
  }
  usage() if (!defined($ARGV[0]) ||
	      (!($ARGV[0] =~ m/^start$/i) &&
	       !($ARGV[0] =~ m/^stop$/i) &&
	       !($ARGV[0] =~ m/^reload$/i) &&
	       !($ARGV[0] =~ m/^report$/i)));

  if (!$opt_no_log)
  {
    w2log("$my_progname log file version $VER; run: ",
	  "$opt_log", 1, 0);
  }
  else
  {
    print "$my_progname log file version $VER; run: ";
    print strftime "%a %b %e %H:%M:%S %Y", localtime;
    print "\n";
  }
  if (($ARGV[0] =~ m/^start$/i) || ($ARGV[0] =~ m/^reload$/i))
  {
    if (!defined(($mysqld= my_which($opt_mysqld))) && $opt_verbose)
    {
      print "WARNING: Couldn't find the default mysqld binary.\n";
      print "Tried: $opt_mysqld\n";
      print "This is OK, if you are using option \"mysqld=...\" in ";
      print "groups [mysqldN] separately for each.\n\n";
    }
    if ($ARGV[0] =~ m/^start$/i) {
      start_mysqlds();
    } elsif ($ARGV[0] =~ m/^reload$/i) {
      reload_mysqlds();
    }
  }
  else
  {
    if (!defined(($mysqladmin= my_which($opt_mysqladmin))) && $opt_verbose)
    {
      print "WARNING: Couldn't find the default mysqladmin binary.\n";
      print "Tried: $opt_mysqladmin\n";
      print "This is OK, if you are using option \"mysqladmin=...\" in ";
      print "groups [mysqldN] separately for each.\n\n";
    }
    if ($ARGV[0] =~ m/^report$/i)
    {
      report_mysqlds();
    }
    else
    {
      stop_mysqlds();
    }
  }
}

#
# Quote word for shell
#

sub quote_shell_word
{
  my ($option)= @_;

  $option =~ s!([^\w=./-])!\\$1!g;
  return $option;
}

sub defaults_for_group
{
  my ($group) = @_;

  return () unless $my_print_defaults_exists;

  my $com= join ' ', 'my_print_defaults', @defaults_options, $group;
  my @defaults = `$com`;
  chomp @defaults;
  return @defaults;
}

####
#### Init log file. Check for appropriate place for log file, in the following
#### order:  my_print_defaults mysqld datadir, /usr/share/mysql
####

sub init_log
{
  foreach my $opt (defaults_for_group('mysqld'))
  {
    if ($opt =~ m/^--datadir=(.*)/ && -d "$1" && -w "$1")
    {
      $logdir= $1;
    }
  }
  if (!defined($logdir))
  {
    $logdir= "/usr/share/mysql" if (-d "/usr/share/mysql" && -w "/usr/share/mysql");
  }
  if (!defined($logdir))
  {
    # Log file was not specified and we could not log to a standard place,
    # so log file be disabled for now.
    if (!$opt_silent)
    {
      print "WARNING: Log file disabled. Maybe directory or file isn't writable?\n";
    }
    $opt_no_log= 1;
  }
  else
  {
    $opt_log= "$logdir/mysqld_multi.log";
  }
}

####
#### Report living and not running MySQL servers
####

sub report_mysqlds
{
  my (@groups, $com, $i, @options, $pec);

  print "Reporting MySQL (Percona Server) servers\n";
  if (!$opt_no_log)
  {
    w2log("\nReporting MySQL (Percona Server) servers","$opt_log",0,0);
  }
  @groups = &find_groups($groupids);
  for ($i = 0; defined($groups[$i]); $i++)
  {
    $com= get_mysqladmin_options($i, @groups);
    $com.= " ping >> /dev/null 2>&1";
    system($com);
    $pec = $? >> 8;
    if ($pec)
    {
      print "MySQL (Percona Server) from group: $groups[$i] is not running\n";
      if (!$opt_no_log)
      {
	w2log("MySQL (Percona Server) from group: $groups[$i] is not running",
	      "$opt_log", 0, 0);
      }
    }
    else
    {
      print "MySQL (Percona Server) from group: $groups[$i] is running\n";
      if (!$opt_no_log)
      {
	w2log("MySQL (Percona Server) from group: $groups[$i] is running",
	      "$opt_log", 0, 0);
      }
    }
  }
  if (!$i)
  {
    print "No groups to be reported (check your GNRs)\n";
    if (!$opt_no_log)
    {
      w2log("No groups to be reported (check your GNRs)", "$opt_log", 0, 0);
    }
  }
}

####
#### start multiple servers
####

sub start_mysqlds()
{
  my (@groups, $com, $tmp, $i, @options, $j, $mysqld_found, $info_sent);

  # We don't need to rebuild the config for each mysqld we might want to start.
  # We rebuild it once when trying to start any servers.  We also require the 
  # config builder script to exist and be executable or we won't try to run it.
  # If the config builder var is undef, we skip it.  If the var is defined but
  # not a real file/executable, we error out and don't try to start.
  #
  my ($cnf_log_message, $cnf_gen_output);
  if (! defined $cnf_builder_path) {
    $cnf_log_message = "Skipping config generation; cnf_builder_path is not defined.";
  } else {
     if (-f -x $cnf_builder_path) {
       $cnf_log_message  = "Regenerating my.cnf via the config builder script...\n";
       $cnf_log_message .= `$cnf_builder_path 2>&1`;
     } else {
       $cnf_log_message = "Config builder not found / not executable - refusing to start.";
     }
  }

  if (!$opt_no_log)
  {
    w2log("\nStarting MySQL (Percona Server) servers\n$cnf_log_message\n","$opt_log",0,0);
  }
  else
  {
    print "\nStarting MySQL (Percona Server) servers\n$cnf_log_message\n";
  }

  @groups = &find_groups($groupids);
  for ($i = 0; defined($groups[$i]); $i++)
  {
    @options = defaults_for_group($groups[$i]);

    $basedir_found= 0; # The default
    $mysqld_found= 1; # The default
    $mysqld_found= 0 if (!length($mysqld));
    $com= "$mysqld";
    my $conflict_message = '';
    for ($j = 0, $tmp= ""; defined($options[$j]); $j++)
    {
      # if this mysqld is already running, don't start it again, include some
      # safety checks to not start it again.  first, try the obvious.
      # - is there a pid file specified?
      # - if so, does it exist?
      # - if it does, does the pid file exist and match the pid of a mysqld ?
      # - if so, don't try to start again.  GTFO.
      if ("--pid_file=" eq substr($options[$j], 0, 11)) {
        my ($junk, $filepath) = split(/\=/, $options[$j]);
        if (-e $filepath) {
          open (FILE, $filepath);
          chomp(my $pid = <FILE>);
          close(FILE);

          # pids are numeric, buddy.
          $pid =~ s/\D//g;

          # if the pid exists, is it mysql?
          if ((-d "/proc/$pid") && (-l "/proc/$pid/exe")) {
            my $cmd_line = readlink("/proc/$pid/exe");
            if (! defined $cmd_line) {
              die "Unable to read the link target for /proc/$pid/exe - you should be running as root."
            } elsif (index($cmd_line, '/mysqld') != -1) {
              $conflict_message = "There is already a MySQL running with pid $pid !";
              last if 1;
            }
          }
        }
      }

      # if for some reason the socket directory is in tmpfs, we need to make sure
      # to recreate it in the event of a system reboot so mysql can start at boot time.
      if ("--socket=" eq substr($options[$j], 0, 9)) {
        my $socket_path = $options[$j];
        $socket_path =~ s/\-\-socket\=//;
        $socket_dir = substr($socket_path, 0, rindex($socket_path, "/"));
        eval {
          mkpath($socket_dir);
          system("chown mysql $socket_dir");
        };
        if ($@) {
          print "FATAL ERROR: Cannot create socket directory: $socket_dir: $!\n";
          exit(1);
        }

        # if the socket file exists, try to use it to ping mysql.  use this to
        # try to handle the situation where there's no pid file but the thing
        # is still running.
        if (-e $socket_path) {
          my $rc = 1;
          eval{
            $rc = system("mysqladmin -S $socket_path ping >/dev/null 2>&1");
          };
          if ($rc == 0) {
            $conflict_message = "There is a running mysqld on socket $socket_path!";
            last if 1;
          }
        }

      }
      if ("--datadir=" eq substr($options[$j], 0, 10)) {
        $datadir = $options[$j];
        $datadir =~ s/\-\-datadir\=//;
        eval { mkpath($datadir) };
        if ($@) {
          print "FATAL ERROR: Cannot create data directory $datadir: $!\n";
          exit(1);
        }
        if (! -d $datadir."/mysql") {
          if (-w $datadir) {
            print "\n\nInstalling new database in $datadir\n\n";
            $install_cmd="/usr/bin/mysql_install_db ";
            $install_cmd.="--user=mysql ";
            $install_cmd.="--datadir=$datadir";
            system($install_cmd);
          } else {
            print "\n";
            print "FATAL ERROR: Tried to create mysqld under group [$groups[$i]],\n";
            print "but the data directory is not writable.\n";
            print "data directory used: $datadir\n";
            exit(1);
          }
        }

        if (! -d $datadir."/mysql") {
          print "\n";
          print "FATAL ERROR: Tried to start mysqld under group [$groups[$i]],\n";
          print "but no data directory was found or could be created.\n";
          print "data directory used: $datadir\n";
          exit(1);
        }
      }

      if ("--mysqladmin=" eq substr($options[$j], 0, 13)) {
        ; # catch this and ignore
      } elsif ("--mysqld=" eq substr($options[$j], 0, 9)) {
        $options[$j]=~ s/\-\-mysqld\=//;
        $com= $options[$j];
        $mysqld_found= 1;
      } elsif ("--basedir=" eq substr($options[$j], 0, 10)) {
        $basedir= $options[$j];
        $basedir =~ s/^--basedir=//;
        $basedir_found= 1;
        $options[$j]= quote_shell_word($options[$j]);
        $tmp.= " $options[$j]";
      } else {
        $options[$j]= quote_shell_word($options[$j]);
        $tmp.= " $options[$j]";
      }
    }

    # was there a running mysqld found?  if so, bail.
    if ($conflict_message) {
      print "FATAL: $conflict_message\n";
      exit(1);
    }

    if ($opt_verbose && $com =~ m/\/(safe_mysqld|mysqld_safe)$/ && !$info_sent)
    {
      print "WARNING: $1 is being used to start mysqld. In this case you ";
      print "may need to pass\n\"ledir=...\" under groups [mysqldN] to ";
      print "$1 in order to find the actual mysqld binary.\n";
      print "ledir (library executable directory) should be the path to the ";
      print "wanted mysqld binary.\n\n";
      $info_sent= 1;
    }
    $com.= $tmp;
    $com.= " >> $opt_log 2>&1" if (!$opt_no_log);
    $com.= " &";

    # find the most recent and available version of jemalloc.
    #
    my $jemalloc_file;
    for (my $i = $max_jemalloc_version; $i >= $min_jemalloc_version; $i--) {
      $jemalloc_file = $jemalloc_base_path . $i;
      if (-f -e $jemalloc_file) {
        print "Launching with jemalloc found in $jemalloc_file\n";
        last if 1;
      } else {
        $jemalloc_file = '';
      }
    }
    # no jemalloc?  that's ok, just launch without it.
    $com = ($jemalloc_file ne '') ? "LD_PRELOAD=$jemalloc_file $com " : $com;

    if (!$mysqld_found)
    {
      print "\n";
      print "FATAL ERROR: Tried to start mysqld under group [$groups[$i]], ";
      print "but no mysqld binary was found.\n";
      print "Please add \"mysqld=...\" in group [mysqld_multi], or add it to ";
      print "group [$groups[$i]] separately.\n";
      exit(1);
    }
    if ($basedir_found)
    {
      $curdir=getcwd();
      chdir($basedir) or die "Can't change to datadir $basedir";
    }
    system($com);
    if ($basedir_found)
    {
      chdir($curdir) or die "Can't change back to original dir $curdir";
    }
  }
  if (!$i && !$opt_no_log)
  {
    w2log("No MySQL (Percona Server) servers to be started (check your GNRs)",
	  "$opt_log", 0, 0);
  }
}

####
#### reload multiple servers
####

sub reload_mysqlds()
{
  my (@groups, $com, $tmp, $i, @options, $j);

  if (!$opt_no_log)
  {
    w2log("\nReloading MySQL servers\n","$opt_log",0,0);
  }
  else
  {
    print "\nReloading MySQL servers\n";
  }
  @groups = &find_groups($groupids);
  for ($i = 0; defined($groups[$i]); $i++)
  {
    $mysqld_server = $mysqld;
    @options = defaults_for_group($groups[$i]);

    for ($j = 0, $tmp= ""; defined($options[$j]); $j++)
    {
      if ("--mysqladmin=" eq substr($options[$j], 0, 13))
      {
        # catch this and ignore
      }
      elsif ("--mysqld=" eq substr($options[$j], 0, 9))
      {
        $options[$j] =~ s/\-\-mysqld\=//;
        $mysqld_server = $options[$j];
      }
      elsif ("--pid-file=" eq substr($options[$j], 0, 11))
      {
        $options[$j] =~ s/\-\-pid-file\=//;
        $pid_file = $options[$j];
      }
    }
    $com = "killproc -p $pid_file -HUP $mysqld_server";
    system($com);

    $com = "touch $pid_file";
    system($com);
  }
  if (!$i && !$opt_no_log)
  {
    w2log("No MySQL servers to be reloaded (check your GNRs)",
         "$opt_log", 0, 0);
  }
}

###
#### stop multiple servers
####

sub stop_mysqlds()
{
  my (@groups, $com, $i, @options);

  if (!$opt_no_log)
  {
    w2log("\nStopping MySQL (Percona Server) servers\n","$opt_log",0,0);
  }
  else
  {
    print "\nStopping MySQL (Percona Server) servers\n";
  }
  @groups = &find_groups($groupids);
  for ($i = 0; defined($groups[$i]); $i++)
  {
    $com= get_mysqladmin_options($i, @groups);
    $com.= " shutdown";
    $com.= " >> $opt_log 2>&1" if (!$opt_no_log);
    $com.= " &";
    system($com);
  }
  if (!$i && !$opt_no_log)
  {
    w2log("No MySQL (Percona Server) servers to be stopped (check your GNRs)",
	  "$opt_log", 0, 0);
  }
}

####
#### Sub function for mysqladmin option parsing
####

sub get_mysqladmin_options
{
  my ($i, @groups)= @_;
  my ($mysqladmin_found, $com, $tmp, $j);

  @options = defaults_for_group($groups[$i]);

  $mysqladmin_found= 1; # The default
  $mysqladmin_found= 0 if (!length($mysqladmin));
  $com = "$mysqladmin";
  $tmp.= $opt_tcp_ip ? " -h 127.0.0.1" : "";
  for ($j = 0; defined($options[$j]); $j++)
  {
    if ("--mysqladmin=" eq substr($options[$j], 0, 13))
    {
      $options[$j]=~ s/\-\-mysqladmin\=//;
      $com= $options[$j];
      $mysqladmin_found= 1;
    }
    elsif ((($options[$j] =~ m/^(\-\-socket\=)(.*)$/) && !$opt_tcp_ip) ||
	   ($options[$j] =~ m/^(\-\-port\=)(.*)$/))
    {
      $tmp.= " $options[$j]";
    }
  }
  if (!$mysqladmin_found)
  {
    print "\n";
    print "FATAL ERROR: Tried to use mysqladmin in group [$groups[$i]], ";
    print "but no mysqladmin binary was found.\n";
    print "Please add \"mysqladmin=...\" in group [mysqld_multi], or ";
    print "in group [$groups[$i]].\n";
    exit(1);
  }
  $com.= $tmp;
  return $com;
}

# Return a list of option files which can be opened.  Similar, but not
# identical, to behavior of my_search_option_files()
sub list_defaults_files
{
  my %opt;
  foreach (@defaults_options)
  {
    return () if /^--no-defaults$/;
    $opt{$1} = $2 if /^--defaults-(extra-file|file)=(.*)$/;
  }

  return ($opt{file}) if exists $opt{file};

  my %seen;  # Don't list the same file more than once
  return grep { defined $_ and not $seen{$_}++ and -f $_ and -r $_ }
              ('/etc/my.cnf',
               '/etc/mysql/my.cnf',
               '/usr/my.cnf',
               ($ENV{MYSQL_HOME} ? "$ENV{MYSQL_HOME}/my.cnf" : undef),
               $opt{'extra-file'},
               ($ENV{HOME} ? "$ENV{HOME}/.my.cnf" : undef));
}


# Takes a specification of GNRs (see --help), and returns a list of matching
# groups which actually are mentioned in a relevant config file
sub find_groups
{
  my ($raw_gids) = @_;

  my %gids;
  my @groups;

  if (defined($raw_gids))
  {
    # Make a hash of the wanted group ids
    foreach my $raw_gid (split ',', $raw_gids)
    {
      # Match 123 or 123-456
      my ($start, $end) = ($raw_gid =~ /^\s*(\d+)(?:\s*-\s*(\d+))?\s*$/);
      $end = $start if not defined $end;
      if (not defined $start or $end < $start or $start < 0)
      {
        print "ABORT: Bad GNR: $raw_gid; see $my_progname --help\n";
        exit(1);
      }

      foreach my $i ($start .. $end)
      {
        # Use $i + 0 to normalize numbers (002 + 0 -> 2)
        $gids{$i + 0}= 1;
      }
    }
  }

  my @defaults_files = list_defaults_files();
  #warn "@{[sort keys %gids]} -> @defaults_files\n";
  foreach my $file (@defaults_files)
  {
    next unless open CONF, "< $file";

    while (<CONF>)
    {
      if (/^\s*\[\s*(mysqld)(\d+)\s*\]\s*$/)
      {
        #warn "Found a group: $1$2\n";
        # Use $2 + 0 to normalize numbers (002 + 0 -> 2)
        if (not defined($raw_gids) or $gids{$2 + 0})
        {
          push @groups, "$1$2";
        }
      }
    }

    close CONF;
  }
  return @groups;
}

####
#### w2log: Write to a logfile.
#### 1.arg: append to the log file (given string, or from a file. if a file,
####        file will be read from $opt_logdir)
#### 2.arg: logfile -name (w2log assumes that the logfile is in $opt_logdir).
#### 3.arg. 0 | 1, if true, print current date to the logfile. 3. arg will
####        be ignored, if 1. arg is a file.
#### 4.arg. 0 | 1, if true, first argument is a file, else a string
####

sub w2log
{
  my ($msg, $file, $date_flag, $is_file)= @_;
  my (@data);

  open (LOGFILE, ">>$opt_log")
    or die "FATAL: w2log: Couldn't open log file: $opt_log\n";

  if ($is_file)
  {
    open (FROMFILE, "<$msg") && (@data=<FROMFILE>) &&
      close(FROMFILE)
	or die "FATAL: w2log: Couldn't open file: $msg\n";
    foreach my $line (@data)
    {
      print LOGFILE "$line";
    }
  }
  else
  {
    print LOGFILE "$msg";
    print LOGFILE strftime "%a %b %e %H:%M:%S %Y", localtime if ($date_flag);
    print LOGFILE "\n";
  }
  close (LOGFILE);
  return;
}

####
#### my_which is used, because we can't assume that every system has the
#### which -command. my_which can take only one argument at a time.
#### Return values: requested system command with the first found path,
#### or undefined, if not found.
####

sub my_which
{
  my ($command) = @_;
  my (@paths, $path);

  return $command if (-f $command && -x $command);
  @paths = split(':', $ENV{'PATH'});
  foreach $path (@paths)
  {
    $path .= "/$command";
    return $path if (-f $path && -x $path);
  }
  return undef();
}


####
#### example
####

sub example
{
  print <<EOF;
# This is an example of a my.cnf file for $my_progname.
# Usually this file is located in home dir ~/.my.cnf or /etc/my.cnf
#
# SOME IMPORTANT NOTES FOLLOW:
#
# 1.COMMON USER
#
#   Make sure that the MySQL user, who is stopping the mysqld services, has
#   the same password to all MySQL servers being accessed by $my_progname.
#   This user needs to have the 'Shutdown_priv' -privilege, but for security
#   reasons should have no other privileges. It is advised that you create a
#   common 'multi_admin' user for all MySQL servers being controlled by
#   $my_progname. Here is an example how to do it:
#
#   GRANT SHUTDOWN ON *.* TO multi_admin\@localhost IDENTIFIED BY 'password'
#
#   You will need to apply the above to all MySQL servers that are being
#   controlled by $my_progname. 'multi_admin' will shutdown the servers
#   using 'mysqladmin' -binary, when '$my_progname stop' is being called.
#
# 2.PID-FILE
#
#   If you are using mysqld_safe to start mysqld, make sure that every
#   MySQL server has a separate pid-file. In order to use mysqld_safe
#   via $my_progname, you need to use two options:
#
#   mysqld=/path/to/mysqld_safe
#   ledir=/path/to/mysqld-binary/
#
#   ledir (library executable directory), is an option that only mysqld_safe
#   accepts, so you will get an error if you try to pass it to mysqld directly.
#   For this reason you might want to use the above options within [mysqld#]
#   group directly.
#
# 3.DATA DIRECTORY
#
#   It is NOT advised to run many MySQL servers within the same data directory.
#   You can do so, but please make sure to understand and deal with the
#   underlying caveats. In short they are:
#   - Speed penalty
#   - Risk of table/data corruption
#   - Data synchronising problems between the running servers
#   - Heavily media (disk) bound
#   - Relies on the system (external) file locking
#   - Is not applicable with all table types. (Such as InnoDB)
#     Trying so will end up with undesirable results.
#
# 4.TCP/IP Port
#
#   Every server requires one and it must be unique.
#
# 5.[mysqld#] Groups
#
#   In the example below the first and the fifth mysqld group was
#   intentionally left out. You may have 'gaps' in the config file. This
#   gives you more flexibility.
#
# 6.MySQL Server User
#
#   You can pass the user=... option inside [mysqld#] groups. This
#   can be very handy in some cases, but then you need to run $my_progname
#   as UNIX root.
#
# 7.A Start-up Manage Script for $my_progname
#
#   In the recent MySQL distributions you can find a file called
#   mysqld_multi.server.sh. It is a wrapper for $my_progname. This can
#   be used to start and stop multiple servers during boot and shutdown.
#
#   You can place the file in /etc/init.d/mysqld_multi.server.sh and
#   make the needed symbolic links to it from various run levels
#   (as per Linux/Unix standard). You may even replace the
#   /etc/init.d/mysql.server script with it.
#
#   Before using, you must create a my.cnf file either in /usr/my.cnf
#   or /root/.my.cnf and add the [mysqld_multi] and [mysqld#] groups.
#
#   The script can be found from support-files/mysqld_multi.server.sh
#   in MySQL distribution. (Verify the script before using)
#

[mysqld_multi]
mysqld     = /usr/bin/mysqld_safe
mysqladmin = /usr/bin/mysqladmin
user       = multi_admin
password   = my_password

[mysqld2]
socket     = /tmp/mysql.sock2
port       = 3307
pid-file   = /var/lib/mysql2/hostname.pid2
datadir    = /var/lib/mysql2
language   = /usr/share/mysql/mysql/english
user       = unix_user1

[mysqld3]
mysqld     = /path/to/mysqld_safe
ledir      = /path/to/mysqld-binary/
mysqladmin = /path/to/mysqladmin
socket     = /tmp/mysql.sock3
port       = 3308
pid-file   = /var/lib/mysql3/hostname.pid3
datadir    = /var/lib/mysql3
language   = /usr/share/mysql/mysql/swedish
user       = unix_user2

[mysqld4]
socket     = /tmp/mysql.sock4
port       = 3309
pid-file   = /var/lib/mysql4/hostname.pid4
datadir    = /var/lib/mysql4
language   = /usr/share/mysql/mysql/estonia
user       = unix_user3
 
[mysqld6]
socket     = /tmp/mysql.sock6
port       = 3311
pid-file   = /var/lib/mysql6/hostname.pid6
datadir    = /var/lib/mysql6
language   = /usr/share/mysql/mysql/japanese
user       = unix_user4
EOF
  exit(0);
}

####
#### usage
####

sub usage
{
  print <<EOF;
$my_progname version $VER by Jani Tolonen

Description:
$my_progname can be used to start, reload, or stop any number of separate
mysqld processes running in different TCP/IP ports and UNIX sockets.

$my_progname can read group [mysqld_multi] from my.cnf file. You may
want to put options mysqld=... and mysqladmin=... there.  Since
version 2.10 these options can also be given under groups [mysqld#],
which gives more control over different versions.  One can have the
default mysqld and mysqladmin under group [mysqld_multi], but this is
not mandatory. Please note that if mysqld or mysqladmin is missing
from both [mysqld_multi] and [mysqld#], a group that is tried to be
used, $my_progname will abort with an error.

$my_progname will search for groups named [mysqld#] from my.cnf (or
the given --defaults-extra-file=...), where '#' can be any positive 
integer starting from 1. These groups should be the same as the regular
[mysqld] group, but with those port, socket and any other options
that are to be used with each separate mysqld process. The number
in the group name has another function; it can be used for starting,
reloading, stopping, or reporting any specific mysqld server.

Usage: $my_progname [OPTIONS] {start|reload|stop|report} [GNR,GNR,GNR...]
or     $my_progname [OPTIONS] {start|reload|stop|report} [GNR-GNR,GNR,GNR-GNR,...]

The GNR means the group number. You can start, reload, stop or report any GNR,
or several of them at the same time. (See --example) The GNRs list can
be comma separated or a dash combined. The latter means that all the
GNRs between GNR1-GNR2 will be affected. Without GNR argument all the
groups found will either be started, reloaded, stopped, or reported. Note that
syntax for specifying GNRs must appear without spaces.

Options:

These options must be given before any others:
--no-defaults      Do not read any defaults file
--defaults-file=...  Read only this configuration file, do not read the
                   standard system-wide and user-specific files
--defaults-extra-file=...  Read this configuration file in addition to the
                   standard system-wide and user-specific files
Using:  @{[join ' ', @defaults_options]}

--example          Give an example of a config file with extra information.
--help             Print this help and exit.
--log=...          Log file. Full path to and the name for the log file. NOTE:
                   If the file exists, everything will be appended.
                   Using: $opt_log
--mysqladmin=...   mysqladmin binary to be used for a server shutdown.
                   Since version 2.10 this can be given within groups [mysqld#]
                   Using: $mysqladmin
--mysqld=...       mysqld binary to be used. Note that you can give mysqld_safe
                   to this option also. The options are passed to mysqld. Just
                   make sure you have mysqld in your PATH or fix mysqld_safe.
                   Using: $mysqld
                   Please note: Since mysqld_multi version 2.3 you can also
                   give this option inside groups [mysqld#] in ~/.my.cnf,
                   where '#' stands for an integer (number) of the group in
                   question. This will be recognised as a special option and
                   will not be passed to the mysqld. This will allow one to
                   start different mysqld versions with mysqld_multi.
--no-log           Print to stdout instead of the log file. By default the log
                   file is turned on.
--silent           Disable warnings.
--tcp-ip           Connect to the MySQL server(s) via the TCP/IP port instead
                   of the UNIX socket. This affects stopping and reporting.
                   If a socket file is missing, the server may still be
                   running, but can be accessed only via the TCP/IP port.
                   By default connecting is done via the UNIX socket.
--verbose          Be more verbose.
--version          Print the version number and exit.
EOF
  exit(0);
}