Permalink
Browse files

Merge branch 'ltrepackage'

  • Loading branch information...
2 parents da558ec + fae8ce9 commit 74f3378f2974656f9f80e22ebde9b3e2e71de1b6 @buzztroll buzztroll committed May 20, 2011
View
33 docs/src/admin/reference.html
@@ -2167,7 +2167,8 @@
Edit <tt class="literal">$NIMBUS_HOME/services/etc/nimbus/workspace-service/other/common.conf</tt>
and change the value of <tt class="literal">propagate.extraargs</tt>:
</p>
- <pre class="panel">propagate.extraargs=$NIMBUS_HOME/lantorrent/bin/lt-request</pre>
+ <pre class="panel">propagate.extraargs=$NIMBUS_HOME/lantorrent/bin/lt-request.sh</pre>
+
<p>
Be sure to expand <tt class="literal">$NIMBUS_HOME</tt> to its full and actual path.
</p>
@@ -2176,18 +2177,38 @@
<li>
<p>Install LANTorrent on VMM</p>
<ul>
- <li>Run <tt class="literal">./vmm-install.sh</tt> on each node.
- Either run it as your workspace control user or specify the workspace
- control user as the first and only argument to the script.
+ <li>Copy lantorrent to the VMM nodes</li>
+ From the source distribution of Nimbus copy the lantorrent
+ directory to all VMM nodes.
+ <li>optional: You may want to setup a python virtual environment for the installation</li>
+ <li>Run <tt class="literal">python setup-vmm.py install</tt> on each node.
+ This will output the xinetd file that you will need to install
+ into /etc/xinetd. Not the <i>user</i> value in the output.
+ this should be the name user as your workspace control user.
</li>
</ul>
</li>
<li>
<p>Install LANTorrent into xinetd</p>
<p>
- The <tt class="literal">vmm-install.sh</tt> script creates the file <tt class="literal">lantorrent</tt>.
- This file is ready to be copied into <tt class="literal">/etc/xinetd.d/</tt>. Once done,
+ The <tt class="literal">setup-vmm.py</tt> script outputs the need xinetd <tt class="literal">lantorrent</tt> file. For example:
+ <pre class="panel">
+ ============== START WITH THE NEXT LINE ==================
+ service lantorrent
+ {
+ type = UNLISTED
+ disable = no
+ socket_type = stream
+ protocol = tcp
+ user = bresnaha
+ wait = no
+ port = 2893
+ server = /home/bresnaha/lt1/bin/ltserver
+ }
+ =============== END WITH THE PREVIOUS LINE =================
+ </pre>
+ This output must be copied into <tt class="literal">/etc/xinetd.d/lantorrent</tt>. Once done,
restart xinetd.
</p>
<pre class="panel">
View
3 home/libexec/nimbusctl.py
@@ -110,7 +110,8 @@
if lantorrent_enabled:
LT_HOME = os.path.join(NIMBUS_HOME, "lantorrent/")
- LT_SERVICE_EXE = os.path.join(LT_HOME, "bin/lt-daemon")
+ os.environ['LANTORRENT_HOME'] = LT_HOME
+ LT_SERVICE_EXE = os.path.join(LT_HOME, "bin/lt-daemon.sh")
if not os.path.exists(LT_SERVICE_EXE):
sys.exit("The services executable does not exist: " +
LT_SERVICE_EXE)
View
4 install
@@ -115,7 +115,9 @@ if [ ! -f $CUMULUS_ENV ]; then
fi
export NIMBUS_HOME
-$NIMBUS_SRC/lantorrent/install.sh $NIMBUS_HOME/lantorrent | tee -a $NIMBUS_HOME/install.log
+source $pyve_path/bin/activate
+export LANTORRENT_HOME=$NIMBUS_HOME/lantorrent
+$NIMBUS_SRC/lantorrent/install.sh $LANTORRENT_HOME | tee -a $NIMBUS_HOME/install.log
rc=$PIPESTATUS
if [ $rc -ne 0 ]; then
echo "lantorrent install failed"
View
30 lantorrent/README.txt
@@ -72,14 +72,32 @@ To install LANTorrent you must take the following steps:
be sure to expand $NIMBUS_HOME to its full and actual path.
3) install lantorrent on VMM
- - run ./vmm-install.sh on each node
- either run it as your workspace control user or specify the workspace
- control user as the first and only argument to the script.
+
+ - run: python setup-vmm.py install. This will output the contents of
+ the xinetd configuration file that you will need in the next step.
4) install lantorrent into xinetd
- - the vmm-install.sh script creates the file lantorrent. This
- file is ready to be copied into /etc/xinetd.d/. Once this is done
- restart xinetd (/etc/init.d/xinetd restart).
+ - the above step outputs and xinetd file like this:
+
+ ============== START WITH THE NEXT LINE ==================
+ service lantorrent
+ {
+ type = UNLISTED
+ disable = no
+ socket_type = stream
+ protocol = tcp
+ user = bresnaha
+ wait = no
+ port = 2893
+ server = /home/bresnaha/lt1/bin/ltserver
+ }
+ =============== END WITH THE PREVIOUS LINE =================
+
+ note the 'user' value. We strongly recommend that it is not 'root'.
+ This user will be the owner of all received files (the nimbus user)
+
+ Copy the output to a file called /etc/xinetd.d/lantorrent. Once
+ this is done restart xinetd (/etc/init.d/xinetd restart).
5) change the propagation method.
- edit the file:
View
12 lantorrent/bin/client.sh
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-
-exec python ./pylantorrent/client.py "${@}"
View
7 lantorrent/bin/fetch.sh
@@ -1,7 +0,0 @@
-#!/bin/bash
-#rm -f /var/lib/eucalyptus/nimbustests/FILE_here
-
-
-
-ssh user04 /home/nimbus/virga/bin/req.sh $ip 2893 /var/lib/eucalyptus/nimbustests/FILE /var/lib/eucalyptus/nimbustests/FILE_here $1 $2
-
View
44 lantorrent/bin/header1
@@ -1,44 +0,0 @@
-{
- "destinations": [
- {
- "block_size": "131072",
- "file": "//home/bresnaha/FIRST.1\n",
- "host": "localhost",
- "id": "f935987a-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
- },
- {
- "block_size": "131072",
- "file": "//home/bresnaha/FIRST.2\n",
- "host": "localhost",
- "id": "f935d22c-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
- },
- {
- "block_size": "131072",
- "file": "//home/bresnaha/FIRST.3\n",
- "host": "localhost",
- "id": "f9360a76-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
- },
- {
- "block_size": "131072",
- "file": "//home/bresnaha/FIRST.4\n",
- "host": "localhost",
- "id": "f9364176-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
- },
- {
- "block_size": "131072",
- "file": "//home/bresnaha/FIRST.5\n",
- "host": "localhost",
- "id": "f9367af6-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
- }
- ],
- "file": "/dev/null",
- "host": "localhost",
- "id": "f93680a0-9da8-11df-a88c-001e378a7ce9",
- "port": "2893"
-}
-EOH : bLSbr4uinA/XfK+eziRHlDJuruE=
View
23 lantorrent/bin/header2
@@ -1,23 +0,0 @@
-{
- "destinations": [
- {
- "file": "/home/bresnaha/test2",
- "host": "localhost",
- "id": "63d90130-9db3-11df-9927-001e378a7ce9",
- "port": "2893",
- "block_size": "2893"
- },
- {
- "file": "/home/bresnaha/test3",
- "host": "localhost",
- "id": "63d90130-9db3-11df-9927-001e378a7ce9",
- "port": "2893",
- "block_size": "2893"
- }
- ],
- "file": "/dev/null",
- "host": "localhost",
- "id": "63d90130-9db3-11df-9927-001e378a7ce9",
- "port": "2893"
-}
-EOH : JWuClmImi9FKTJL/5D0CpqBk4Qg=
View
20 lantorrent/bin/lt-daemon
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# this is the only end that will be located with the nimbus service
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-# so that we pick up the ini file
-export LANTORRENT_HOME=$pypath
-cd ../
-export NIMBUS_HOME=`pwd`
-source $NIMBUS_HOME/ve/bin/activate
-
-exec python $LANTORRENT_HOME/pylantorrent/daemon.py "${@}"
-
-
View
18 lantorrent/bin/lt-request
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# this is the only end that will be located with the nimbus service
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-# so that we pick up the ini file
-export LANTORRENT_HOME=$pypath
-cd ../
-export NIMBUS_HOME=`pwd`
-source $NIMBUS_HOME/ve/bin/activate
-
-exec python $LANTORRENT_HOME/pylantorrent/request.py "${@}"
View
13 lantorrent/bin/lt.sh
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-
-export LANTORRENT_HOME=$pypath
-exec python ./pylantorrent/server.py "${@}"
View
7 lantorrent/etc/exe.in
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+dir=`dirname $0`
+cd $dir/..
+export LANTORRENT_HOME=`pwd`
+
+exec @PGMNAME@ "${@}"
View
2 lantorrent/etc/lantorrent.inet.in
@@ -7,5 +7,5 @@ service @SERVICENAME@
user = @WHO@
wait = no
port = @PORT@
- server = @LANTORRENT_HOME@/bin/lt.sh
+ server = @SERVER@
}
View
6 lantorrent/etc/lt.ini
@@ -2,7 +2,7 @@
password=nimbus
[log]
-file=@LANTORRENT_HOME@/log/lantorrent.@PGM@.log
-level=debug
+file=@LANTORRENT_HOME@/lantorrent.@PGM@.log
+level=info
[db]
-file=@LANTORRENT_HOME@/etc/req.db
+file=@LANTORRENT_HOME@/req.db
View
15 lantorrent/etc/lt.sql
@@ -1,15 +0,0 @@
-create table requests(
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- rid varchar(36) UNIQUE NOT NULL,
- src_filename varchar(1024) NOT NULL,
- dst_filename varchar(1024) NOT NULL,
- xfer_id varchar(64),
- hostname varchar(1024) NOT NULL,
- port INTEGER NOT NULL DEFAULT 2893,
- state INTEGER DEFAULT 0,
- message TEXT,
- entry_time TIMESTAMP,
- attempt_count INTEGER DEFAULT 0,
- UNIQUE(hostname, port, dst_filename)
-);
-
View
61 lantorrent/install.sh
@@ -1,55 +1,32 @@
#!/bin/bash
-if [ "X$1" == "X-h" ]; then
- echo "<install dir>"
- exit 0
-fi
if [ "X$1" == "X" ]; then
- echo "please specify a target directory"
+ echo "you must provide a target installation directory"
exit 1
fi
-echo "-----------------------------------------------------------------"
-echo " Installing LANTorrent"
-echo "-----------------------------------------------------------------"
+export LANTORRENT_HOME=$1
-src_dir=`dirname $0`
-cd $src_dir
-rc=$?
-if [ $rc -ne 0 ]; then
- echo "could not change directories"
- exit $rc
-fi
-src_dir=`pwd`
+pylib=`python -c 'import sys; print sys.prefix'`
-installdir=$1
-mkdir $installdir
-cd $installdir
-installdir=`pwd`
+echo "Installing into the python libraries into $pylibs..."
+echo "Installing wrapper scripts into $LANTORRENT_HOME"
+mkdir -p $LANTORRENT_HOME/bin
+mkdir -p $LANTORRENT_HOME/etc
+dir=`dirname $0`
+cd $dir
-echo ""
-echo "Copying from $src_dir to $installdir"
-cp -r $src_dir/* $installdir/
-rc=$?
-if [ $rc -ne 0 ]; then
- echo "failed to copy over lantorrent"
- exit $rc
+python setup.py install
+if [ $? -ne 0 ]; then
+ exit 1
fi
-cd $installdir
-rc=$?
-if [ $rc -ne 0 ]; then
- echo "could not change to the installation directory"
- exit $rc
-fi
+ltrequest="$pylib/bin/ltrequest"
+ltdaemon="$pylib/bin/ltdaemon"
-rm -f etc/req.db
-sqlite3 etc/req.db < etc/lt.sql
-rc=$?
-if [ $rc -ne 0 ]; then
- echo "could not create the database"
- exit $rc
-fi
-echo "Success."
-exit 0
+sed "s^@PGMNAME@^$ltrequest^" etc/exe.in > $LANTORRENT_HOME/bin/lt-request.sh
+sed "s^@PGMNAME@^$ltdaemon^" etc/exe.in > $LANTORRENT_HOME/bin/lt-daemon.sh
+
+chmod 755 $LANTORRENT_HOME/bin/lt-request.sh
+chmod 755 $LANTORRENT_HOME/bin/lt-daemon.sh
View
49 lantorrent/perf/boto_many.py
@@ -1,49 +0,0 @@
-#!/usr/bin/python
-
-
-import sys
-import string
-import random
-import os
-import sys
-import sys
-import ConfigParser
-from ConfigParser import SafeConfigParser
-import time
-import tempfile
-import traceback
-import filecmp
-import logging
-import shlex
-from optparse import SUPPRESS_HELP
-import boto
-from boto.s3.connection import OrdinaryCallingFormat
-from boto.s3.connection import VHostCallingFormat
-from boto.s3.connection import SubdomainCallingFormat
-from boto.s3.connection import S3Connection
-from boto.ec2.connection import EC2Connection
-
-
-def main(argv=sys.argv[1:]):
-
- host = argv[0]
- imagename = argv[1]
- count = int(argv[2])
-
- print "setting up ec2 connection"
- ec2conn = EC2Connection(s3id, pw, host=host, port=8444, debug=2)
- ec2conn.host = host
-
- print "getting a reference to the remote image"
- image = ec2conn.get_image(imagename)
- print "running %d instances" % (count)
- res = image.run(min_count=count, max_count=count)
- print "stoping all instances"
- res.stop_all()
-
- return 0
-
-if __name__ == "__main__":
- rc = main()
- sys.exit(rc)
-
View
7 lantorrent/perf/cc_tests/cmd.sh
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-histdir=`mktemp -d`
-cmd="/home/bresnaha/nimbus-cloud-client-016/bin/cloud-client.sh --name lenny-vm.raw --hours .1 --run --history-dir $histdir"
-$cmd
-rm -rf $histdir
-
View
19 lantorrent/perf/cc_tests/counter.sh
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-tm_file=$1
-cnts="4 8 16 32 64"
-trials=4
-
-touch $tm_file
-for c in $cnts
-do
- for i in `seq 1 $trials`
- do
- echo $i
- /usr/bin/time --append -o $tm_file --format "$c %e" ./timer.sh $c
- sleep 90
- rm -rf /home/bresnaha/nimbus-cloud-client-016/history/*
- sleep 90
- done
-done
-
View
14 lantorrent/perf/cc_tests/timer.sh
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-cmd="./cmd.sh"
-max=$1
-pid_list=""
-for i in `seq 1 $max`
-do
- touch run.log
- $cmd | tee -a run.log &
- pid_list="$pid_list $!"
-done
-
-wait
-
View
180 lantorrent/pylantorrent/__init__.py
@@ -3,6 +3,9 @@
import logging
import traceback
from ConfigParser import SafeConfigParser
+from optparse import OptionParser
+from optparse import SUPPRESS_HELP
+
import hmac
try:
from hashlib import sha1 as sha
@@ -48,29 +51,36 @@ def create_endpoint_entry(host, dest_files, data_size, port=2893, block_size=128
class VConfig(object):
def __init__(self):
- self.set_defaults()
+
if 'LANTORRENT_HOME' not in os.environ:
- emsg = "the env LAN_TORRENT_HOME must be set"
- self.lt_home = os.path.expanduser("lantorrent")
+ self.lt_home = os.path.expanduser("~/.lantorrent")
+ emsg = "the env LAN_TORRENT_HOME is not set, using %s" % (self.lt_home)
+ try:
+ os.mkdir(self.lt_home)
+ except:
+ pass
+ self.set_defaults()
+ logging.basicConfig(filename=self.logfile, level=self.log_level,
+ format="%(asctime)s - %(levelname)s - %(message)s")
log(logging.WARNING, emsg)
else:
- self.lt_home = os.environ['LANTORRENT_HOME']
+ self.lt_home = os.environ['LANTORRENT_HOME']
+ self.set_defaults()
ini_file = os.path.join(self.lt_home, "etc/lt.ini")
try:
self.load_settings(ini_file)
except:
emsg = "failed to load %s, using defaults" % (ini_file)
log(logging.WARNING, emsg, traceback)
- logging.basicConfig(filename=self.logfile, level=self.log_level,
- format="%(asctime)s - %(levelname)s - %(message)s")
- log(logging.WARNING, "logging to %s at %d" % (self.logfile, self.log_level))
+ logging.basicConfig(filename=self.logfile, level=self.log_level,
+ format="%(asctime)s - %(levelname)s - %(message)s")
def set_defaults(self):
self.pw = "nimbus"
- self.logfile = "lantorrent.log"
+ self.logfile = "%s/lantorrent.log" % (self.lt_home)
self.log_level = logging.DEBUG
- self.dbfile = None
+ self.dbfile = "%s/reqs.db" % (self.lt_home)
self.db_error_max = 10
self.insert_delay = 30
@@ -107,3 +117,155 @@ def get_auth_hash(header_str):
return auth_hash
+class cbOpts(object):
+
+ def __init__(self, long, short, description, default, vals=None, range=None, flag=None, count=False, long_help=False):
+ self.long = "--" + long.replace('_', '-')
+ self.dest = long
+ self.short = "-" + short
+ self.vals = vals
+ self.default = default
+ self.flag = flag
+ self.range = range
+ self.description = description
+ self.count = count
+ self.long_help = long_help
+
+ def get_error_msg(self):
+ if self.flag != None:
+ emsg = "The option %s | %s is a flag" % (self.short, self.long)
+ return emsg
+
+ if self.range != None:
+ emsg = "The option %s | %s must be between %s and %s" % (self.short, self.long, self.range[0], self.range[1])
+ return emsg
+
+ if self.vals != None:
+ emsg = "The value for %s | %s must be: { " % (self.short, self.long)
+ delim = ""
+ for v in self.vals:
+ emsg = emsg + delim + str(v)
+ delim = " | "
+ emsg = emsg + "}"
+
+ return emsg
+
+ return "Error"
+
+ def validate(self, options):
+
+ try:
+ val = getattr(options, self.dest)
+ except:
+ emsg = self.get_error_msg()
+ raise Exception(emsg)
+
+ if val == None:
+ return
+ if self.flag != None:
+ return
+ if self.range != None:
+ if len(self.range) == 2:
+ if float(val) == -1.0:
+ if float(self.range[0]) != -1.0 and float(self.range[1]) != -1.0:
+ raise Exception("you specified a value out of range")
+
+ else:
+ return
+
+ if (float(val) < float(self.range[0]) and float(self.range[0]) != -1.0) or (float(val) > float(self.range[1]) and float(self.range[1]) != -1.0):
+ emsg = self.get_error_msg()
+ raise Exception(emsg)
+ return
+
+ if self.vals != None:
+ for v in self.vals:
+ if val == v:
+ return
+
+ emsg = self.get_error_msg()
+ raise Exception(emsg)
+
+ def get_description(self):
+ if self.range != None:
+ msg = self.description + " : between %s - %s" % (self.range[0], self.range[1])
+ return msg
+
+ if self.vals != None:
+ msg = self.description + " : {"
+ delim = ""
+ for v in self.vals:
+ msg = msg + delim + str(v)
+ delim = " | "
+ msg = msg + "}"
+
+ return msg
+
+ return self.description
+
+ def add_opt(self, parser):
+ if self.flag != None:
+ if self.default:
+ a = "store_false"
+ else:
+ a = "store_true"
+ parser.add_option(self.short, self.long, dest=self.dest, default=self.default,
+ action=a,
+ help=self.get_description())
+ return
+
+ if self.count:
+ parser.add_option(self.short, self.long, dest=self.dest,
+ default=self.default,
+ action="count",
+ help=self.get_description())
+ return
+
+ parser.add_option(self.short, self.long, dest=self.dest,
+ default=self.default, type="string",
+ help=self.get_description())
+
+
+__author__ = 'bresnaha'
+
+def get_default_options(u):
+ global Version
+ version = "%prog " + (Version)
+ parser = OptionParser(usage=u, version=version)
+
+ all_opts = []
+
+ opt = cbOpts("batch", "b", "Set to batch mode for machine parsing", False, flag=True)
+ all_opts.append(opt)
+ opt = cbOpts("quiet", "q", "Display no output", False, flag=True)
+ all_opts.append(opt)
+ opt = cbOpts("verbose", "V", "Display much output, repeat to increase level", 1, count=True)
+ all_opts.append(opt)
+ opt = cbOpts("instream", "I", SUPPRESS_HELP, None)
+ all_opts.append(opt)
+ opt = cbOpts("outstream", "O", SUPPRESS_HELP, None)
+ all_opts.append(opt)
+
+ return (parser, all_opts)
+
+
+def parse_args(p, all_opts, argv):
+
+ for o in all_opts:
+ o.add_opt(p)
+ (options, args) = p.parse_args(args=argv)
+
+ if options.outstream == None:
+ options.out_file = sys.stdout
+ else:
+ options.out_file = open(options.outstream, "w")
+ if options.instream == None:
+ options.in_file = sys.stdin
+ else:
+ options.in_file = open(options.instream, "r")
+
+ for o in all_opts:
+ o.validate(options)
+
+ return (options, args)
+
View
4 lantorrent/pylantorrent/client.py
@@ -176,6 +176,10 @@ def main(argv=sys.argv[1:]):
return 0
if __name__ == "__main__":
+ if 'LANTORRENT_HOME' not in os.environ:
+ msg = "The env LANTORRENT_HOME must be set"
+ print msg
+ raise Exception(msg)
rc = main()
sys.exit(rc)
View
8 lantorrent/pylantorrent/daemon.py
@@ -4,6 +4,7 @@
from socket import *
import logging
import pylantorrent
+from pylantorrent.db import LantorrentDB
from pylantorrent.server import LTServer
from pylantorrent.client import LTClient
try:
@@ -153,6 +154,9 @@ def main(argv=sys.argv[1:]):
pylantorrent.log(logging.INFO, "enter %s" % (sys.argv[0]))
+ # use sqlaclh to make sure the db is there
+ x = LantorrentDB("sqlite:///%s" % pylantorrent.config.dbfile)
+ x.close()
con_str = pylantorrent.config.dbfile
#con = sqlite3.connect(con_str, isolation_level="EXCLUSIVE")
con = sqlite3.connect(con_str, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
@@ -172,5 +176,9 @@ def main(argv=sys.argv[1:]):
return 0
if __name__ == "__main__":
+ if 'LANTORRENT_HOME' not in os.environ:
+ msg = "The env LANTORRENT_HOME must be set"
+ print msg
+ raise Exception(msg)
rc = main()
sys.exit(rc)
View
76 lantorrent/pylantorrent/db.py
@@ -0,0 +1,76 @@
+#### right now this is just used to create the db
+
+import sqlalchemy
+from sqlalchemy.orm import mapper
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Table
+from sqlalchemy import Integer
+from sqlalchemy import String, MetaData, Sequence
+from sqlalchemy import Column
+from sqlalchemy import types
+from datetime import datetime
+import sqlite3
+
+metadata = MetaData()
+
+request_table = Table('requests', metadata,
+ Column('id', Integer, Sequence('event_id_seq')),
+ Column('rid', String(36), nullable=False, unique=True),
+ Column('src_filename', String(1024), nullable=False),
+ Column('dst_filename', String(1024), nullable=False, primary_key = True),
+ Column('xfer_id', String(64)),
+ Column('hostname', String(1024), nullable=False, primary_key = True),
+ Column('port', Integer, nullable=False, default=2893, primary_key = True),
+ Column('state', Integer, nullable=False, default=0),
+ Column('message', String(4096), nullable=True),
+ Column('entry_time', types.TIMESTAMP(), default=datetime.now()),
+ Column('attempt_count', Integer, nullable=False, default=0),
+ )
+
+class RequestTable(object):
+ def __init__(self):
+ self.id = None
+ self.rid = None
+ self.src_filename = None
+ self.dst_filename = None
+ self.xfer_id = None
+ self.hostname = None
+ self.port = None
+ self.state = None
+ self.message = None
+ self.entry_time = None
+ self.attempt_count = None
+
+mapper(RequestTable, request_table)
+
+class LantorrentDB(object):
+
+ def __init__(self, dburl):
+
+ # this is raw sql deal that i used
+ self._engine = sqlalchemy.create_engine(dburl,
+ connect_args={'detect_types': sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES},
+ native_datetime=True
+ )
+
+ metadata.create_all(self._engine)
+ self._Session = sessionmaker(bind=self._engine)
+ self._session = self._Session()
+
+
+ def db_obj_add(self, obj):
+ self._session.add(obj)
+
+ def db_commit(self):
+ self._session.commit()
+
+ def db_rollback(self):
+ self._session.rollback()
+
+ def raw_sql(self, sql):
+ con = self._session.connection()
+ res = con.execute(sql)
+ return list(res)
+
+ def close(self):
+ self._session.close()
View
2 lantorrent/pylantorrent/nosetests/__init__.py
@@ -0,0 +1,2 @@
+__author__ = 'bresnaha'
+
View
31 lantorrent/pylantorrent/request.py
@@ -1,36 +1,34 @@
import sqlite3
-import sys
-import os
from socket import *
import logging
import pylantorrent
+from pylantorrent.db import LantorrentDB
from pylantorrent.server import LTServer
from pylantorrent.client import LTClient
+import os
try:
import json
except ImportError:
import simplejson as json
-import traceback
-import uuid
import time
import random
import datetime
-import pynimbusauthz
-from pynimbusauthz.cmd_opts import cbOpts
+from pylantorrent import cbOpts
+import sys
def setup_options(argv):
u = """[options]
Submit a transfer request
"""
- (parser, all_opts) = pynimbusauthz.get_default_options(u)
+ (parser, all_opts) = pylantorrent.get_default_options(u)
opt = cbOpts("nonblock", "n", "Do not block waiting for completion", False, flag=True)
all_opts.append(opt)
opt = cbOpts("reattach", "a", "Reattach", None)
all_opts.append(opt)
- (o, args) = pynimbusauthz.parse_args(parser, all_opts, argv)
+ (o, args) = pylantorrent.parse_args(parser, all_opts, argv)
return (o, args, parser)
@@ -93,6 +91,8 @@ def delete_rid(con, rid):
time.sleep(random.random() * 2.0)
def request(argv, con):
+ if len(argv) < 4:
+ raise Exception("You must provide 4 arguments: <src file> <dst file> <a uuid for this request> <the contanct string of the receiving nodes lt server>")
src_filename = argv[0]
dst_filename = argv[1]
# the user provides the rid. that way we know they have it to look
@@ -115,8 +115,8 @@ def request(argv, con):
port = 2893
now = datetime.datetime.now()
- i = "insert into requests(src_filename, dst_filename, hostname, port, rid, entry_time) values (?, ?, ?, ?, ?, ?)"
- data = (src_filename, dst_filename, host, port, rid, now,)
+ i = "insert into requests(src_filename, dst_filename, hostname, port, rid, entry_time, state, attempt_count) values (?, ?, ?, ?, ?, ?, ?, ?)"
+ data = (src_filename, dst_filename, host, port, rid, now, 0, 0, )
error_ctr = 0
while True:
@@ -155,6 +155,10 @@ def main(argv=sys.argv[1:]):
(o, args, p) = setup_options(argv)
+ # use sqlaclh to make sure the db is there
+ x = LantorrentDB("sqlite:///%s" % pylantorrent.config.dbfile)
+ x.close()
+
con_str = pylantorrent.config.dbfile
con = sqlite3.connect(con_str, isolation_level="EXCLUSIVE")
@@ -182,12 +186,17 @@ def main(argv=sys.argv[1:]):
delete_rid(con, rid)
msg = "%d,%s,%s" % (rc, str(done), message)
- pynimbusauthz.print_msg(o, 0, msg)
+ print msg
return rc
if __name__ == "__main__":
+ if 'LANTORRENT_HOME' not in os.environ:
+ msg = "The env LANTORRENT_HOME must be set"
+ print msg
+ raise Exception(msg)
+
rc = main()
# always return 0. an non 0 return code will mean an ssh error
sys.exit(0)
View
15 lantorrent/pylantorrent/server.py
@@ -5,10 +5,7 @@
import pylantorrent
from pylantorrent.ltException import LTException
from pylantorrent.ltConnection import *
-try:
- import json
-except ImportError:
- import simplejson as json
+import simplejson as json
import traceback
import hashlib
@@ -59,7 +56,9 @@ def clean_up(self, force=False):
for f in self.created_files:
try:
pylantorrent.log(logging.DEBUG, "deleting file %s" % (f))
- if f != "/dev/null":
+ # dont delete /dev/null (or any other dev really)
+ ndx = f.strip().find("/dev")
+ if ndx != 0:
os.remove(f)
except:
pass
@@ -194,7 +193,6 @@ def _rename_files(self, requests_a):
def main(argv=sys.argv[1:]):
pylantorrent.log(logging.INFO, "server starting")
- v = None
rc = 1
v = LTServer(sys.stdin, sys.stdout)
try:
@@ -215,6 +213,11 @@ def main(argv=sys.argv[1:]):
return rc
if __name__ == "__main__":
+ if 'LANTORRENT_HOME' not in os.environ:
+ msg = "The env LANTORRENT_HOME must be set"
+ print msg
+ raise Exception(msg)
+
rc = main()
sys.exit(rc)
View
101 lantorrent/setup-vmm.py
@@ -0,0 +1,101 @@
+__author__ = 'bresnaha'
+
+import os
+from setuptools import setup, find_packages
+import sys
+
+Version = "0.2"
+
+if float("%d.%d" % sys.version_info[:2]) < 2.4:
+ sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3])
+ sys.stderr.write("lantorrent requires Python 2.5 or newer.\n")
+ sys.exit(1)
+
+setup(name='lantorrent',
+ version=Version,
+ description='An Open Source network protocol for broadcasting large files.',
+ author='Nimbus Development Team',
+ author_email='workspace-user@globus.org',
+ url='http://www.nimbusproject.org/',
+ packages=[ 'pylantorrent', 'pylantorrent.nosetests'],
+ entry_points = {
+ 'console_scripts': [
+ 'ltserver = pylantorrent.server:main',
+ 'ltclient = pylantorrent.client:main',
+ ],
+
+ },
+ download_url ="http://www.nimbusproject.org/downloads/lantorrent-%s.tar.gz" % (Version),
+ keywords = "multicast broadcast network protocol transfer",
+ long_description="""
+Fast multicast of large files (designed for VM images)
+""",
+ license="Apache2",
+ install_requires = ["simplejson == 2.1"],
+
+ include_package_data = True,
+# data_files = test_plans,
+ package_data = {},
+
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Intended Audience :: End Users/Desktop',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: System Administrators',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: MacOS :: MacOS X',
+ 'Operating System :: Microsoft :: Windows',
+ 'Operating System :: POSIX',
+ 'Operating System :: POSIX :: Linux',
+ 'Programming Language :: Python',
+ 'Topic :: System :: Clustering',
+ 'Topic :: System :: Distributed Computing',
+ ],
+ )
+
+# generate the xinitd file and setup messages
+print ""
+
+lt_home = os.path.expanduser("~/.lantorrent")
+if 'LANTORRENT_HOME' in os.environ:
+ lt_home = os.environ['LANTORRENT_HOME']
+
+ini_template = "etc/lt.ini"
+dir = os.path.dirname(__file__)
+if dir:
+ ini_template = dir + "/" + ini_template
+ini_template = os.path.abspath(ini_template)
+dst = os.path.abspath(lt_home + "/etc/lt.ini")
+
+print ""
+print "======================="
+print "Copying the configuration file %s to %s" % (ini_template, dst)
+cp_cmd = "cp %s %s" % (ini_template, dst)
+rc = os.system(cp_cmd)
+if rc != 0:
+ raise Exception('Failed to copy the config file %s' % (cp_cmd))
+
+print "===================================="
+print "Generating the xinetd file"
+
+import getpass
+who = getpass.getuser()
+server = sys.prefix + "/bin/ltserver"
+
+xinetd_template = "etc/lantorrent.inet.in"
+dir = os.path.dirname(__file__)
+if dir:
+ xinetd_template = dir + "/" + xinetd_template
+
+f = open(xinetd_template, "r")
+
+sed_list = [("@PORT@", "2893"), ("@SERVICENAME@", "lantorrent"), ("@WHO@", who), ("@SERVER@", server)]
+print "please place the following in /etc/xinetd.d/lantorrent"
+print "============== START WITH THE NEXT LINE =================="
+for line in f:
+ for (old, new) in sed_list:
+ line = line.replace(old, new)
+ sys.stdout.write(line)
+
+print "=============== END WITH THE PREVIOUS LINE ================="
View
80 lantorrent/setup.py
@@ -0,0 +1,80 @@
+__author__ = 'bresnaha'
+
+import os
+from setuptools import setup, find_packages
+import sys
+
+
+Version = "0.2"
+
+if float("%d.%d" % sys.version_info[:2]) < 2.5:
+ sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3])
+ sys.stderr.write("lantorrent requires Python 2.5 or newer.\n")
+ sys.exit(1)
+
+setup(name='lantorrent',
+ version=Version,
+ description='An Open Source network protocol for broadcasting large files.',
+ author='Nimbus Development Team',
+ author_email='workspace-user@globus.org',
+ url='http://www.nimbusproject.org/',
+ packages=[ 'pylantorrent', 'pylantorrent.nosetests'],
+ entry_points = {
+ 'console_scripts': [
+ 'ltdaemon = pylantorrent.daemon:main',
+ 'ltserver = pylantorrent.server:main',
+ 'ltrequest = pylantorrent.request:main',
+ 'ltclient = pylantorrent.client:main',
+ ],
+
+ },
+ download_url ="http://www.nimbusproject.org/downloads/lantorrent-%s.tar.gz" % (Version),
+ keywords = "multicast broadcast network protocol transfer",
+ long_description="""
+Fast multicast of large files (designed for VM images)
+""",
+ license="Apache2",
+ install_requires = ["simplejson == 2.1", "sqlalchemy == 0.6"],
+
+ include_package_data = True,
+# data_files = test_plans,
+ package_data = {},
+
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Intended Audience :: End Users/Desktop',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: System Administrators',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: MacOS :: MacOS X',
+ 'Operating System :: Microsoft :: Windows',
+ 'Operating System :: POSIX',
+ 'Operating System :: POSIX :: Linux',
+ 'Programming Language :: Python',
+ 'Topic :: System :: Clustering',
+ 'Topic :: System :: Distributed Computing',
+ ],
+ )
+
+lt_home = os.path.expanduser("~/.lantorrent")
+if 'LANTORRENT_HOME' in os.environ:
+ lt_home = os.environ['LANTORRENT_HOME']
+
+ini_template = "etc/lt.ini"
+dir = os.path.dirname(__file__)
+if dir:
+ ini_template = dir + "/" + ini_template
+ini_template = os.path.abspath(ini_template)
+dst = os.path.abspath(lt_home + "/etc/lt.ini")
+
+print ""
+print "======================="
+print "Copying the configuration file %s to %s" % (ini_template, dst)
+cp_cmd = "cp %s %s" % (ini_template, dst)
+rc = os.system(cp_cmd)
+if rc != 0:
+ raise Exception('Failed to copy the config file %s' % (cp_cmd))
+
+
+print "Lantorrent is now installed into your python environment. We recommend using the LANTORRENT_HOME environment variable to specify the location of configuration and log files. If this env is not set then ~/.lantorrent is used."
View
15 lantorrent/bin/make_lt_server.sh → lantorrent/tests/make_lt_server.sh
@@ -1,21 +1,16 @@
#!/bin/bash
+if [ "X$LANTORRENT_HOME" == "X" ]; then
+ echo "Please set the env LANTORRENT_HOME"
+ exit 1
+fi
+
count=$1
if [ "X$1" == "X" ]; then
count=1
fi
pidfile=$2
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-# so that we pick up the ini file
-export LANTORRENT_HOME=$pypath
who=`whoami`
delim=""
View
1 lantorrent/tests/simple_test.py
@@ -61,4 +61,3 @@ def test_block_sizes(self):
-
View
28 lantorrent/bin/test.sh → lantorrent/tests/test.sh
@@ -1,30 +1,24 @@
#!/bin/bash
+if [ "X$LANTORRENT_HOME" == "X" ]; then
+ echo "Please set the env LANTORRENT_HOME"
+ exit 1
+fi
+if [ "X$NIMBUS_HOME" == "X" ]; then
+ echo "Please set the env NIMBUS_HOME"
+ exit 1
+fi
+cd $LANTORRENT_HOME
+
# most users will not have xinetd in their path even if it is installed
export PATH=/usr/sbin/:$PATH
-dir=`dirname $0`
-cd $dir/..
-pypath=`pwd`
-
-cd ../
-export NIMBUS_HOME=`pwd`
-source $NIMBUS_HOME/ve/bin/activate
-
-if [ "X${PYTHONPATH}" == "X" ]; then
- export PYTHONPATH=$pypath
-else
- export PYTHONPATH=$pypath:${PYTHONPATH}
-fi
-# so that we pick up the ini file
-export LANTORRENT_HOME=$pypath
pidfile=`mktemp`
$LANTORRENT_HOME/bin/make_lt_server.sh 4 $pidfile
xinet_pid=`cat $pidfile`
echo "xinet on $xinet_pid"
-cd $LANTORRENT_HOME
-$dir/lt-daemon &
+ltdaemon &
ltd_pid=$!
trap "kill $xinet_pid $ltd_pid; sleep 10; kill -9 $xinet_pid $ltd_pid" EXIT
View
26 lantorrent/vmm-install.sh
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-if [ "X$1" == "X-h" ]; then
- echo "usage: vmm-install.sh [<user to run the service under>]"
- exit 0
-fi
-if [ "X$1" == "X" ]; then
- who=`whoami`
-else
- who=$1
-fi
-
-src_dir=`dirname $0`
-cd $src_dir
-LANTORRENT_HOME=`pwd`
-
-sed -e "s/@PORT@/2893/" -e "s/@SERVICENAME@/lantorrent/" -e "s/@WHO@/$who/" -e "s^@LANTORRENT_HOME@^$LANTORRENT_HOME^" etc/lantorrent.inet.in > lantorrent
-rc=$?
-if [ $rc -ne 0 ]; then
- echo "Failed to create the inetd entry"
- exit $rc
-fi
-
-echo "Install complete. Copy $src_dir/lantorrent into /etc/xinetd.d/ and restart xinetd"
-
-exit 0
View
2 service/service/java/source/etc/workspace-service/other/common.conf
@@ -65,5 +65,5 @@ caches.dir=$NIMBUS_HOME/services/var/nimbus/caches
msgsinks.dir=$NIMBUS_HOME/services/var/nimbus/msg-sinks
-#propagate.extraargs=$NIMBUS_HOME/lantorrent/bin/lt-request
+#propagate.extraargs=$NIMBUS_HOME/lantorrent/bin/lt-request.sh
propagate.extraargs=

0 comments on commit 74f3378

Please sign in to comment.