Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

ctx-agent/ from 2009-01-10

  • Loading branch information...
commit 67dffb673896f68b914857dfd4742f59997c252f 1 parent 80433cc
@timf timf authored
Showing with 7,224 additions and 0 deletions.
  1. +17 −0 ctx-agent/ctx-scripts/0-etchosts.sh
  2. +37 −0 ctx-agent/ctx-scripts/1-ipandhost/nfsclient
  3. +35 −0 ctx-agent/ctx-scripts/1-ipandhost/nfsserver
  4. +23 −0 ctx-agent/ctx-scripts/1-ipandhost/testrole1
  5. +23 −0 ctx-agent/ctx-scripts/1-ipandhost/testrole2
  6. +23 −0 ctx-agent/ctx-scripts/1-ipandhost/testrole3
  7. +33 −0 ctx-agent/ctx-scripts/1-ipandhost/torquemaster
  8. +37 −0 ctx-agent/ctx-scripts/1-ipandhost/torqueslave
  9. +89 −0 ctx-agent/ctx-scripts/2-thishost/publicnic
  10. +24 −0 ctx-agent/ctx-scripts/3-data/gridmap
  11. +25 −0 ctx-agent/ctx-scripts/3-data/startcontainer
  12. +27 −0 ctx-agent/ctx-scripts/3-data/testdata
  13. +31 −0 ctx-agent/ctx-scripts/4-restarts/nfsclient
  14. +49 −0 ctx-agent/ctx-scripts/4-restarts/nfsserver
  15. +21 −0 ctx-agent/ctx-scripts/4-restarts/testrole2
  16. +20 −0 ctx-agent/ctx-scripts/4-restarts/torquemaster
  17. +21 −0 ctx-agent/ctx-scripts/4-restarts/torqueslave
  18. +94 −0 ctx-agent/ctx-scripts/5-thishost-finalize/publicnic
  19. +21 −0 ctx-agent/ctx-scripts/clean.sh
  20. +9 −0 ctx-agent/ctx-scripts/problem.sh
  21. +178 −0 ctx-agent/ctx/ctx.conf
  22. +10 −0 ctx-agent/ctx/launch.sh
  23. +37 −0 ctx-agent/ctx/lib/err-template-001.xml
  24. +45 −0 ctx-agent/ctx/lib/err-template-002.xml
  25. +40 −0 ctx-agent/ctx/lib/ok-template-001.xml
  26. +40 −0 ctx-agent/ctx/lib/ok-template-002.xml
  27. +141 −0 ctx-agent/ctx/lib/pylib/embeddedET/ElementInclude.py
  28. +196 −0 ctx-agent/ctx/lib/pylib/embeddedET/ElementPath.py
  29. +1,254 −0 ctx-agent/ctx/lib/pylib/embeddedET/ElementTree.py
  30. +230 −0 ctx-agent/ctx/lib/pylib/embeddedET/HTMLTreeBuilder.py
  31. +103 −0 ctx-agent/ctx/lib/pylib/embeddedET/SgmlopXMLTreeBuilder.py
  32. +144 −0 ctx-agent/ctx/lib/pylib/embeddedET/SimpleXMLTreeBuilder.py
  33. +279 −0 ctx-agent/ctx/lib/pylib/embeddedET/SimpleXMLWriter.py
  34. +6 −0 ctx-agent/ctx/lib/pylib/embeddedET/TidyHTMLTreeBuilder.py
  35. +128 −0 ctx-agent/ctx/lib/pylib/embeddedET/TidyTools.py
  36. +113 −0 ctx-agent/ctx/lib/pylib/embeddedET/XMLTreeBuilder.py
  37. +30 −0 ctx-agent/ctx/lib/pylib/embeddedET/__init__.py
  38. +34 −0 ctx-agent/ctx/lib/retr-template-001.xml
  39. +42 −0 ctx-agent/ctx/lib/retr-template-002.xml
  40. +3,494 −0 ctx-agent/ctx/lib/workspace_ctx_retrieve.py
  41. +21 −0 ctx-agent/ctx/sample-rc-local-entry.txt
View
17 ctx-agent/ctx-scripts/0-etchosts.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# This configures the local /etc/hosts file with all members of the context.
+# Don't change or delete this script unless you know what you are doing.
+
+echo ""
+echo "etchosts script"
+echo "IP: $1"
+echo "Short hostname: $2"
+echo "Hostname: $3"
+
+echo "$1 $3 $2" >> /etc/hosts
+
+
+exit 0
+
+
View
37 ctx-agent/ctx-scripts/1-ipandhost/nfsclient
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "NFS client required: we are being told this node requires an NFS client,"
+echo "therefore it will be playing the role of NFS server."
+echo ""
+
+# This script configures the NFS server exports policy to allow the one client
+# it's now hearing about to import the exported volumes.
+
+echo "NFS CLIENT IP: $1"
+echo "NFS CLIENT Short hostname: $2"
+echo "NFS CLIENT Hostname: $3"
+
+EXPORTS="/etc/exports"
+
+# append IP to every non-empty line in EXPORTS
+RESULT="`sed -e \"s/ \$/ $1(rw,no_root_squash) /\" $EXPORTS`"
+
+echo "$RESULT" > "$EXPORTS"
+
+exit 0
+
+
View
35 ctx-agent/ctx-scripts/1-ipandhost/nfsserver
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "NFS server required: we are being told this node requires an NFS server,"
+echo "therefore it will be playing the role of an NFS client."
+echo ""
+
+# This script configures the fstab file to list an import from this server node
+# we are hearing about. Repeat below for each volume to import. If you have
+# multiple NFS servers, each exporting something different, make sure you have
+# DIFFERENT role names for each (like "nfsserver1" "nfsserver2" etc).
+
+echo "NFS SERVER IP: $1"
+echo "NFS SERVER Short hostname: $2"
+echo "NFS SERVER Hostname: $3"
+
+echo "$1:/home /home nfs tcp,rsize=32768,wsize=32768,intr,soft,user,exec 0 0" >> /etc/fstab
+echo "$1:/etc/grid-security/certificates /etc/grid-security/certificates nfs tcp,rsize=32768,wsize=32768,intr,soft,user,exec 0 0" >> /etc/fstab
+
+exit 0
+
+
View
23 ctx-agent/ctx-scripts/1-ipandhost/testrole1
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "TESTROLE1 IP: $1"
+echo "TESTROLE1 Short hostname: $2"
+echo "TESTROLE1 Hostname: $3"
+
+echo -e "Testing: testrole1 is provided by host $3 \n" >> /tmp/testrole1
+
+
View
23 ctx-agent/ctx-scripts/1-ipandhost/testrole2
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "TESTROLE2 IP: $1"
+echo "TESTROLE2 Short hostname: $2"
+echo "TESTROLE2 Hostname: $3"
+
+echo -e "Testing: testrole2 is provided by host $3 \n" >> /tmp/testrole2
+
+
View
23 ctx-agent/ctx-scripts/1-ipandhost/testrole3
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "TESTROLE3 IP: $1"
+echo "TESTROLE3 Short hostname: $2"
+echo "TESTROLE3 Hostname: $3"
+
+echo -e "Testing: testrole3 is provided by host $3 \n" >> /tmp/testrole3
+
+
View
33 ctx-agent/ctx-scripts/1-ipandhost/torquemaster
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "Torque master required: we are being told this node requires a Torque"
+echo "master, therefore it will be playing the role of Torque slave."
+echo ""
+
+echo "TORQUEMASTER IP: $1"
+echo "TORQUEMASTER Short hostname: $2"
+echo "TORQUEMASTER Hostname: $3"
+
+# Configure the Torque server name to enslave to. Replace the file contents,
+# do not append since only one can be handled. It would be a misconfiguration
+# to have more than one node playing the torquemaster role in a context.
+
+echo "\$pbsserver $3" > /var/spool/torque/mom_priv/config
+echo "$3" > /var/spool/torque/server_name
+
+exit 0
+
View
37 ctx-agent/ctx-scripts/1-ipandhost/torqueslave
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 1-ipandhost scripts are called when the context broker tells this node it #
+# *requires* to know about nodes playing the role. If this node is told it #
+# requires to know about nodes playing the "xyz" role, then if a script #
+# called "xyz" lives in this directory, it will be called with IP, short #
+# hostname and hostname (args $1, $2, and $3 respectively) of the node that #
+# *provides* the required "xyz" role. #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "Torque slave required: we are being told this node requires a Torque"
+echo "slave, therefore it will be playing the role of Torque master."
+echo ""
+
+echo "TORQUESLAVE IP: $1"
+echo "TORQUESLAVE Short hostname: $2"
+echo "TORQUESLAVE Hostname: $3"
+
+# Add this node we are hearing about to the nodes list. For every node that
+# provides the torqueslave role, this script will be called. Therefore we
+# APPEND (not replace) to this nodes file:
+
+echo "$2 np=2" >> /var/spool/torque/server_priv/nodes
+
+
+# This is a trick to send a message to another script:
+touch /root/this_node_is_torque_master
+
+exit 0
+
+
View
89 ctx-agent/ctx-scripts/2-thishost/publicnic
@@ -0,0 +1,89 @@
+#!/bin/sh
+
+# ************************************************************************ #
+# 2-thishost scripts are called with *this* node's IP, short hostname and #
+# fully qualified hostname. It can be useful in multi NIC situations #
+# especially. #
+# ************************************************************************ #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+# NOTE: The name of this script must correspond to the interface name that
+# the context broker knows, not the local interface name which may or
+# may not match.
+#
+# The context agent can only handle two NICs at most. By convention, if
+# there is more than one NIC, the nics need to be labelled "publicnic" or
+# "localnic" as defined by the metadata server. On EC2, the publicnic
+# is the public IP address that NATs the VM (it does not correspond to
+# an actual NIC in the VM). Note again that the labelling is NOT the
+# interface name in the VM but rather the labels in the contextualization
+# document where different roles may be played by different IP addresses
+# (and they are labelled with NIC names).
+
+echo "publicnic thishost script: configuring local programs before restarts"
+
+echo "This IP: $1"
+echo "This short local hostname: $2"
+echo "This FQDN: $3"
+
+
+# SSH host based authentication was configured already, restart SSH to load
+# new config.
+
+/etc/init.d/sshd restart
+
+# We're overloading torque master role to imply other head node setups.
+# Could do something explicit too.
+if [ ! -e "/root/this_node_is_torque_master" ]; then
+ exit 0
+fi
+
+
+# For torque, replace entire contents of these files
+echo "$3" > /var/spool/torque/server_name
+echo "root@$3" > /var/spool/torque/server_priv/acl_svr/operators
+
+
+# Create a self-signed cert for onboard CA operations
+/root/bin/ca.sh $3 > /root/safe/host.0
+if [ $? -ne 0 ]; then
+ echo "failed to make onboard trust root"
+ exit 1
+fi
+
+
+# node should trust itself. We need to get the cert hash for the
+# filename first:
+HASH=`openssl x509 -in /root/safe/host.0 -hash -noout`
+CAFILE="/etc/grid-security/certificates/$HASH.0"
+cp /root/safe/host.0 $CAFILE
+echo "New 'CA' cert: $CAFILE"
+
+# for user pickup
+cp $CAFILE /root/certs/
+
+function makefile (){
+ touch $3
+ chown $1 $3
+ chmod $2 $3
+}
+
+makefile root 444 /etc/grid-security/hostcert.pem
+cp /root/safe/host.0 /etc/grid-security/hostcert.pem
+makefile root 400 /etc/grid-security/hostkey.pem
+cp /root/safe/host.key /etc/grid-security/hostkey.pem
+
+makefile globus 444 /etc/grid-security/containercert.pem
+cp /root/safe/host.0 /etc/grid-security/containercert.pem
+makefile globus 400 /etc/grid-security/containerkey.pem
+cp /root/safe/host.key /etc/grid-security/containerkey.pem
+
+
+
+
+
+
+
View
24 ctx-agent/ctx-scripts/3-data/gridmap
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# ******************************************************************* #
+# 3-data scripts have filenames that correspond to data names in the #
+# context. If this node is told it requires data 'xyz' and a script #
+# in the 3-data directory bears the name 'xyz' then it is called. #
+# The data value has already been written out to a temporary file. #
+# The absolute path of that file is given as argument $1 to this #
+# script for you to do as you please. #
+# ******************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "'gridmap' data"
+echo "Filename with data value: $1"
+
+echo "Copying that to /etc/grid-security/grid-mapfile"
+
+cp $1 /etc/grid-security/grid-mapfile
+
+
View
25 ctx-agent/ctx-scripts/3-data/startcontainer
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# ******************************************************************* #
+# 3-data scripts have filenames that correspond to data names in the #
+# context. If this node is told it requires data 'xyz' and a script #
+# in the 3-data directory bears the name 'xyz' then it is called. #
+# The data value has already been written out to a temporary file. #
+# The absolute path of that file is given as argument $1 to this #
+# script for you to do as you please. #
+# ******************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "Filename with data value: $1"
+
+echo "This is being used as an example of how one could do a trigger. The "
+echo "presence of this data in the context tells us to indeed start the "
+echo "container."
+
+# This "sends a message" to another script:
+touch /root/do_startcontainer
+
View
27 ctx-agent/ctx-scripts/3-data/testdata
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# ******************************************************************* #
+# 3-data scripts have filenames that correspond to data names in the #
+# context. If this node is told it requires data 'xyz' and a script #
+# in the 3-data directory bears the name 'xyz' then it is called. #
+# The data value has already been written out to a temporary file. #
+# The absolute path of that file is given as argument $1 to this #
+# script for you to do as you please. #
+# ******************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "'testdata' example data script"
+echo "Filename with data value: $1"
+
+echo "Copying that to /tmp/testdata-received"
+
+cp $1 /tmp/testdata-received
+
+# That tempfile will contain whatever was put into the context under
+# this dataname.
+
+
View
31 ctx-agent/ctx-scripts/4-restarts/nfsclient
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 4-restarts scripts are a little counter-intuitive. If *requiring a role* #
+# implies this node *plays a role*, you can base restarts off of this. #
+# For example, if "pbsslave" is a required role, it implies this node is a #
+# PBS server and the pbsslave restart script could restart the PBS daemon #
+# since it's called after all the configuration is done (ipandhost scripts) #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+# This scripts is named "nfsclient" and is called if this node *requires*
+# "nfsclient" role. If it requires "nfsclient" then it's an NFS server,
+# restart the NFS server.
+
+echo "NFS client role needed: restart NFS server"
+
+/etc/init.d/nfs restart
+
+if [ $? -ne 0 ]; then
+ sleep 3
+ /etc/init.d/nfs restart
+ exit $?
+fi
+
+exit 0
+
+
View
49 ctx-agent/ctx-scripts/4-restarts/nfsserver
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 4-restarts scripts are a little counter-intuitive. If *requiring a role* #
+# implies this node *plays a role*, you can base restarts off of this. #
+# For example, if "pbsslave" is a required role, it implies this node is a #
+# PBS server and the pbsslave restart script could restart the PBS daemon #
+# since it's called after all the configuration is done (ipandhost scripts) #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+# This scripts is named "nfsserver" and is called if this node *requires*
+# "nfsserver" role. If it requires "nfsserver" then it's an NFS client,
+# so initiate the client mounts here.
+
+echo "NFS server role needed, initiating NFS client mounts"
+
+# /etc/init.d/netfs restart and cousins are not as helpful as this.
+
+# Should add a feature in the future that fails out after so many attempts etc.
+
+function mountone (){
+ echo "Trying to mount $1"
+
+ MOUNTED="no"
+
+ while [ "$MOUNTED" = "no" ]; do
+ mount $1
+ if [ $? -eq 0 ]; then
+ MOUNTED="yes"
+ echo "$1 is now mounted"
+ else
+ echo "$1 not mounted, sleeping 3 seconds"
+ sleep 3
+ fi
+ done
+}
+
+mountone "/home"
+mountone "/etc/grid-security/certificates"
+
+
+
+
+
+
View
21 ctx-agent/ctx-scripts/4-restarts/testrole2
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 4-restarts scripts are a little counter-intuitive. If *requiring a role* #
+# implies this node *plays a role*, you can base restarts off of this. #
+# For example, if "pbsslave" is a required role, it implies this node is a #
+# PBS server and the pbsslave restart script could restart the PBS daemon #
+# since it's called after all the configuration is done (ipandhost scripts) #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo ""
+echo "This node was told it required testrole2, so this "testrole2" restarts"
+echo "script triggers booting up something or other in response to this fact."
+
+# See real examples for when you'd want to make a restarts script.
+
+
View
20 ctx-agent/ctx-scripts/4-restarts/torquemaster
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 4-restarts scripts are a little counter-intuitive. If *requiring a role* #
+# implies this node *plays a role*, you can base restarts off of this. #
+# For example, if "pbsslave" is a required role, it implies this node is a #
+# PBS server and the pbsslave restart script could restart the PBS daemon #
+# since it's called after all the configuration is done (ipandhost scripts) #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo "Torque server role needed, restart Torque client"
+
+/etc/init.d/pbs_mom start
+
+exit 0
+
View
21 ctx-agent/ctx-scripts/4-restarts/torqueslave
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# ************************************************************************* #
+# 4-restarts scripts are a little counter-intuitive. If *requiring a role* #
+# implies this node *plays a role*, you can base restarts off of this. #
+# For example, if "pbsslave" is a required role, it implies this node is a #
+# PBS server and the pbsslave restart script could restart the PBS daemon #
+# since it's called after all the configuration is done (ipandhost scripts) #
+# ************************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+echo "Torque client role needed, restart Torque server"
+
+/etc/init.d/pbs_sched start
+/etc/init.d/pbs_server start
+
+exit 0
+
View
94 ctx-agent/ctx-scripts/5-thishost-finalize/publicnic
@@ -0,0 +1,94 @@
+#!/bin/sh
+
+# ********************************************************************* #
+# 5-thishost-finalize scripts are called with *this* node's IP, short #
+# hostname and fully qualified hostname. They are called *after* #
+# 4-restarts (as the numbering system implies). They are somewhat #
+# esoteric, but can be useful if a local program depends on one of the #
+# things restarted in 4-restarts. #
+# ********************************************************************* #
+
+RELDIR=`dirname $0`
+ABSDIR=`cd $RELDIR; pwd`
+echo "Hello from \"$ABSDIR/$0\""
+
+# NOTE: The name of this script must correspond to the interface name that
+# the context broker knows, not the local interface name which may or
+# may not match.
+#
+# The context agent can only handle two NICs at most. By convention, if
+# there is more than one NIC, the nics need to be labelled "publicnic" or
+# "localnic" as defined by the metadata server. On EC2, the publicnic
+# is the public IP address that NATs the VM (it does not correspond to
+# an actual NIC in the VM). Note again that the labelling is NOT the
+# interface name in the VM but rather the labels in the contextualization
+# document where different roles may be played by different IP addresses
+# (and they are labelled with NIC names).
+
+echo "publicnic thishost finalize: configuring or launching something after restarts"
+
+echo "This IP: $1"
+echo "This short local hostname: $2"
+echo "This FQDN: $3"
+
+# We're overloading torque master role to imply other head node setups.
+# Could do something explicit by way of the context roles too, but this
+# works just fine (assuming torque master is always same node as GT).
+if [ ! -e "/root/this_node_is_torque_master" ]; then
+ exit 0
+fi
+
+
+# cluster-001:
+
+# Start postgresql before Globus
+/etc/init.d/postgresql start
+if [ $? -ne 0 ]; then
+ echo "problem starting postgresql"
+ exit 1
+fi
+
+# We have put a keyword everywhere that the local hostname needs to appear
+# in the GT4 configs. This sed command replaces keyword with local FQDN.
+
+REPLACEKEYWORD="THISHOST_REPLACE_FQDN"
+GL="/opt/gt4"
+
+function replace (){
+ echo "Adding local FQDN to: $2"
+
+ OUTPUTFILE=/root/safe/`uuidgen`
+
+ sed "s/$REPLACEKEYWORD/$1/g" $2 > $OUTPUTFILE
+
+ # Trick while you're setting contextualization up: manually call this
+ # script and run the sed replacement backwards,to get back to the replacement
+ # tokens before the image is --save'd back to the repository.
+ # sed "s/$1/$REPLACEKEYWORD/g" $2 > $OUTPUTFILE
+
+ cp $OUTPUTFILE $2
+}
+
+replace $3 $GL/etc/gram-service/jndi-config.xml
+replace $3 $GL/etc/gram-service/globus_gram_fs_map_config.xml
+replace $3 $GL/etc/globus-job-manager.conf
+replace $3 $GL/etc/globus_wsrf_core/server-config.wsdd
+replace $3 $GL/client-config.wsdd
+
+
+# see 3-data/startcontainer
+if [ ! -e "/root/do_startcontainer" ]; then
+ exit 0
+fi
+
+# Start GT container.
+/etc/init.d/globus start
+if [ $? -ne 0 ]; then
+ echo "problem starting globus container"
+ exit 1
+fi
+
+
+
+
+
View
21 ctx-agent/ctx-scripts/clean.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# You can put commands that 'reset' the VM for contextualization
+# here and then call this before launch.sh in your rc.local (like
+# the sample clusters do).
+
+cp /etc/hosts.clean /etc/hosts
+cp /etc/exports.clean /etc/exports
+
+cat /dev/null > /etc/hosts.equiv
+
+# wipe the torque nodes file, ipandhost script will fill it with all
+# compute nodes
+cat /dev/null > /var/spool/torque/server_priv/nodes
+
+rm -f /root/this_node_is_torque_master
+
+cat /dev/null > /opt/nimbus/ctxlog.txt
+rm -rf /opt/nimbus/ctx/tmp/
+mkdir /opt/nimbus/ctx/tmp/
+
View
9 ctx-agent/ctx-scripts/problem.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+echo "Problem script running... shutting down."
+touch /tmp/contextualization_problem_script_ran
+
+echo "Poweroff could be run here. Currently commented out."
+# /sbin/poweroff
+
+
View
178 ctx-agent/ctx/ctx.conf
@@ -0,0 +1,178 @@
+[sshd]
+
+# Absolute path only, the sshd host key
+generatedkey: /etc/ssh/ssh_host_rsa_key.pub
+
+# SSHd hosts config is adjusted directly by this program right now, adds
+# equiv hostnames and pubkeys for host based authorization across the whole
+# contextualization context.
+
+hostbasedconfig: /etc/hosts.equiv
+knownhostsconfig: /etc/ssh/ssh_known_hosts
+
+
+[reginst]
+
+#### Regular instantiation
+
+# Path to metadata server URL file
+path: /var/nimbus-metadata-server-url
+
+# Comma separated names of possible identity nics (do NOT use lo, for example).
+# These are REAL local interface names that may be present (each is checked for
+# an IP configuration).
+nicnames: eth0, eth1
+
+
+[systempaths]
+
+# These can be relative or absolute paths.
+hostname: hostname
+curl: curl
+
+
+# ***NOTE: it is unlikely you need to alter the configurations below this line
+
+[taskpaths]
+
+#### Calling order (this is explained in more detail below).
+#### 0-etchosts
+#### 1-ipandhost
+#### 2-thishost
+#### 3-data
+#### 4-restarts
+#### 5-thishostfinalize
+
+
+#### 0-etchosts
+#
+# every identity seen is always sent to etchosts
+# arg1: IP
+# arg2: short hostname
+# arg3: FQDN
+
+etchosts: /opt/nimbus/ctx-scripts/0-etchosts.sh
+
+
+#### 1-ipandhost
+#
+# Directory where the scripts live that match the required role names.
+# See samples for more explanation.
+#
+# These role scripts receive:
+# arg1: IP
+# arg2: short hostname
+# arg3: FQDN
+
+ipandhostdir: /opt/nimbus/ctx-scripts/1-ipandhost
+
+
+#### 2-thishost
+#
+# "thishost" scripts are called with network information known about the
+# host this program is running on.
+#
+# Each script receives:
+# arg1: IP
+# arg2: Short local hostname
+# arg3: FQDN
+#
+# The names of the scripts in this directory must correspond to the interface
+# that the context broker knows about, not the local interface which may not
+# match.
+#
+# Particular scripts may be absent. The entire directory configuration
+# may also be absent.
+
+thishostdir: /opt/nimbus/ctx-scripts/2-thishost
+
+
+#### 3-data
+#
+# The opaque data directory contains scripts that match provided data names.
+# If data fields exist in the context, the data is written to a file and
+# that file absolute path is sent as only argument to the scripts.
+# The scripts are called after 'thishost' but before 'restarts'.
+
+datadir: /opt/nimbus/ctx-scripts/3-data
+
+
+#### 4-restarts
+#
+# The restart directory contains scripts that match provided roles.
+#
+# After all role information has been added via the ipandhostdir script AND
+# after the "thishost" scripts have successfully run, this program will call
+# the restart script for each required role it knows about (presumably to
+# restart the service now that config has changed).
+#
+# No arguments are sent.
+#
+# It is OK for the required role to not have a script in this directory.
+
+restartdir: /opt/nimbus/ctx-scripts/4-restarts
+
+
+#### 5-thishostfinalize
+#
+# The "thishostfinalize" scripts are called with network information known
+# about the host this program is running on. It is called AFTER the restart
+# scripts are successfully called.
+#
+# Each script receives:
+# arg1: IP
+# arg2: Short local hostname
+# arg3: FQDN
+#
+# The names of the scripts in this directory must correspond to the interface
+# that the context broker knows about, not the local interface which may not
+# match.
+#
+# Particular scripts may be absent. The entire directory configuration
+# may also be absent.
+
+thishostfinalizedir: /opt/nimbus/ctx-scripts/5-thishost-finalize
+
+
+# "problem" script
+# In case of problems, could call poweroff. This script will be called after
+# an attempt to notify the service of the error (that notification provides
+# a log of the run to the context broker).
+#
+# Must be configured if "--poweroff" (-p) argument is used, will not be
+# consulted if that argument is not used.
+
+problemscript: /opt/nimbus/ctx-scripts/problem.sh
+
+
+[ctxservice]
+
+# logfile of the run
+# If config is missing, no log will be written and nothing will be sent to
+# service for error reporting.
+logfilepath: /opt/nimbus/ctxlog.txt
+
+# Directory where the program can write temporary files
+scratchspacedir: /opt/nimbus/ctx/tmp
+
+retr_template: /opt/nimbus/ctx/lib/retr-template-001.xml
+retr_template2: /opt/nimbus/ctx/lib/retr-template-002.xml
+err_template: /opt/nimbus/ctx/lib/err-template-001.xml
+err_template2: /opt/nimbus/ctx/lib/err-template-002.xml
+ok_template: /opt/nimbus/ctx/lib/ok-template-001.xml
+ok_template2: /opt/nimbus/ctx/lib/ok-template-002.xml
+
+
+
+[ec2]
+
+#### EC2 instantiation (alternative to regular method)
+
+# URLs for the Amazon REST instance data API
+localhostnameURL: http://169.254.169.254/2007-01-19/meta-data/local-hostname
+publichostnameURL: http://169.254.169.254/2007-01-19/meta-data/public-hostname
+localipURL: http://169.254.169.254/2007-01-19/meta-data/local-ipv4
+publicipURL: http://169.254.169.254/2007-01-19/meta-data/public-ipv4
+publickeysURL: http://169.254.169.254/2007-01-19/meta-data/public-keys/
+userdataURL: http://169.254.169.254/2007-01-19/user-data
+
View
10 ctx-agent/ctx/launch.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+BASEDIR=`dirname $0`
+
+PYTHONPATH="$BASEDIR/lib/pylib:$PYTHONPATH"
+export PYTHONPATH
+
+python $BASEDIR/lib/workspace_ctx_retrieve.py -c $BASEDIR/ctx.conf -z -t
+
+
View
37 ctx-agent/ctx/lib/err-template-001.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/03/addressing">
+ <soapenv:Header>
+ <wsa:MessageID soapenv:mustUnderstand="0">uuid:REPLACE_MESSAGE_ID</wsa:MessageID>
+ <wsa:To soapenv:mustUnderstand="0">REPLACE_SERVICE_URL</wsa:To>
+ <wsa:Action soapenv:mustUnderstand="0">
+ http://www.globus.org/2008/12/nimbus/contextualization/NimbusContextBrokerPortType/retrieveRequest</wsa:Action>
+ <wsa:From soapenv:mustUnderstand="0">
+ <wsa:Address>
+ http://schemas.xmlsoap.org/ws/2004/03/addressing/role/anonymous</wsa:Address>
+ </wsa:From>
+ <ns2:NimbusContextBrokerKey xmlns:ns2="http://www.globus.org/2008/12/nimbus/contextualization"
+ soapenv:mustUnderstand="0">REPLACE_RESOURCE_KEY</ns2:NimbusContextBrokerKey>
+ </soapenv:Header>
+ <soapenv:Body>
+ <errorExiting xmlns="http://www.globus.org/2008/12/nimbus/contextualization">
+
+ <ns2:identity xmlns:ns2="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns2:interface>REPLACE_IFACE_NAME</ns2:interface>
+ <ns2:ip>REPLACE_IFACE_IP</ns2:ip>
+ <ns2:hostname>REPLACE_IFACE_HOSTNAME</ns2:hostname>
+ <ns2:pubkey>REPLACE_IFACE_SSH_KEY</ns2:pubkey>
+ </ns2:identity>
+
+ <ns11:errorcode xmlns:ns11="http://www.globus.org/2008/12/nimbus/ctxtypes">REPLACE_ERRORCODE</ns11:errorcode>
+ <ns12:message xmlns:ns12="http://www.globus.org/2008/12/nimbus/ctxtypes">
+ <![CDATA[REPLACE_ERRORMSG]]>
+ </ns12:message>
+
+ </errorExiting>
+ </soapenv:Body>
+</soapenv:Envelope>
+
View
45 ctx-agent/ctx/lib/err-template-002.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/03/addressing">
+ <soapenv:Header>
+ <wsa:MessageID soapenv:mustUnderstand="0">uuid:REPLACE_MESSAGE_ID</wsa:MessageID>
+ <wsa:To soapenv:mustUnderstand="0">REPLACE_SERVICE_URL</wsa:To>
+ <wsa:Action soapenv:mustUnderstand="0">
+ http://www.globus.org/2008/12/nimbus/contextualization/NimbusContextBrokerPortType/retrieveRequest</wsa:Action>
+ <wsa:From soapenv:mustUnderstand="0">
+ <wsa:Address>
+ http://schemas.xmlsoap.org/ws/2004/03/addressing/role/anonymous</wsa:Address>
+ </wsa:From>
+ <ns2:NimbusContextBrokerKey xmlns:ns2="http://www.globus.org/2008/12/nimbus/contextualization"
+ soapenv:mustUnderstand="0">REPLACE_RESOURCE_KEY</ns2:NimbusContextBrokerKey>
+ </soapenv:Header>
+ <soapenv:Body>
+ <errorExiting xmlns="http://www.globus.org/2008/12/nimbus/contextualization">
+
+ <ns1:identity xmlns:ns1="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns1:interface>REPLACE_IFACE_NAME</ns1:interface>
+ <ns1:ip>REPLACE_IFACE_IP</ns1:ip>
+ <ns1:hostname>REPLACE_IFACE_HOSTNAME</ns1:hostname>
+ <ns1:pubkey>REPLACE_IFACE_SSH_KEY</ns1:pubkey>
+ </ns1:identity>
+
+ <ns2:identity xmlns:ns2="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns2:interface>REPLACE_IFACE2_NAME</ns2:interface>
+ <ns2:ip>REPLACE_IFACE2_IP</ns2:ip>
+ <ns2:hostname>REPLACE_IFACE2_HOSTNAME</ns2:hostname>
+ <ns2:pubkey>REPLACE_IFACE2_SSH_KEY</ns2:pubkey>
+ </ns2:identity>
+
+ <ns11:errorcode xmlns:ns11="http://www.globus.org/2008/12/nimbus/ctxtypes">REPLACE_ERRORCODE</ns11:errorcode>
+ <ns12:message xmlns:ns12="http://www.globus.org/2008/12/nimbus/ctxtypes">
+ <![CDATA[REPLACE_ERRORMSG]]>
+ </ns12:message>
+
+ </errorExiting>
+ </soapenv:Body>
+</soapenv:Envelope>
+
View
40 ctx-agent/ctx/lib/ok-template-001.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/03/addressing">
+ <soapenv:Header>
+ <wsa:MessageID soapenv:mustUnderstand="0">uuid:REPLACE_MESSAGE_ID</wsa:MessageID>
+ <wsa:To soapenv:mustUnderstand="0">REPLACE_SERVICE_URL</wsa:To>
+ <wsa:Action soapenv:mustUnderstand="0">
+ http://www.globus.org/2008/12/nimbus/contextualization/NimbusContextBrokerPortType/retrieveRequest</wsa:Action>
+ <wsa:From soapenv:mustUnderstand="0">
+ <wsa:Address>
+ http://schemas.xmlsoap.org/ws/2004/03/addressing/role/anonymous</wsa:Address>
+ </wsa:From>
+ <ns2:NimbusContextBrokerKey xmlns:ns2="http://www.globus.org/2008/12/nimbus/contextualization"
+ soapenv:mustUnderstand="0">REPLACE_RESOURCE_KEY</ns2:NimbusContextBrokerKey>
+ </soapenv:Header>
+ <soapenv:Body>
+ <okExiting xmlns="http://www.globus.org/2008/12/nimbus/contextualization">
+
+ <ns1:identity xmlns:ns1="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns1:interface>REPLACE_IFACE_NAME</ns1:interface>
+ <ns1:ip>REPLACE_IFACE_IP</ns1:ip>
+ <ns1:hostname>REPLACE_IFACE_HOSTNAME</ns1:hostname>
+ <ns1:pubkey>REPLACE_IFACE_SSH_KEY</ns1:pubkey>
+ </ns1:identity>
+
+ <ns2:identity xmlns:ns2="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns2:interface>REPLACE_IFACE2_NAME</ns2:interface>
+ <ns2:ip>REPLACE_IFACE2_IP</ns2:ip>
+ <ns2:hostname>REPLACE_IFACE2_HOSTNAME</ns2:hostname>
+ <ns2:pubkey>REPLACE_IFACE2_SSH_KEY</ns2:pubkey>
+ </ns2:identity>
+
+ </okExiting>
+ </soapenv:Body>
+</soapenv:Envelope>
+
View
40 ctx-agent/ctx/lib/ok-template-002.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
+xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/03/addressing">
+ <soapenv:Header>
+ <wsa:MessageID soapenv:mustUnderstand="0">uuid:REPLACE_MESSAGE_ID</wsa:MessageID>
+ <wsa:To soapenv:mustUnderstand="0">REPLACE_SERVICE_URL</wsa:To>
+ <wsa:Action soapenv:mustUnderstand="0">
+ http://www.globus.org/2008/12/nimbus/contextualization/NimbusContextBrokerPortType/retrieveRequest</wsa:Action>
+ <wsa:From soapenv:mustUnderstand="0">
+ <wsa:Address>
+ http://schemas.xmlsoap.org/ws/2004/03/addressing/role/anonymous</wsa:Address>
+ </wsa:From>
+ <ns2:NimbusContextBrokerKey xmlns:ns2="http://www.globus.org/2008/12/nimbus/contextualization"
+ soapenv:mustUnderstand="0">REPLACE_RESOURCE_KEY</ns2:NimbusContextBrokerKey>
+ </soapenv:Header>
+ <soapenv:Body>
+ <okExiting xmlns="http://www.globus.org/2008/12/nimbus/contextualization">
+
+ <ns1:identity xmlns:ns1="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns1:interface>REPLACE_IFACE_NAME</ns1:interface>
+ <ns1:ip>REPLACE_IFACE_IP</ns1:ip>
+ <ns1:hostname>REPLACE_IFACE_HOSTNAME</ns1:hostname>
+ <ns1:pubkey>REPLACE_IFACE_SSH_KEY</ns1:pubkey>
+ </ns1:identity>
+
+ <ns2:identity xmlns:ns2="http://www.globus.org/2008/12/nimbus/ctxdescription">
+
+ <ns2:interface>REPLACE_IFACE2_NAME</ns2:interface>
+ <ns2:ip>REPLACE_IFACE2_IP</ns2:ip>
+ <ns2:hostname>REPLACE_IFACE2_HOSTNAME</ns2:hostname>
+ <ns2:pubkey>REPLACE_IFACE2_SSH_KEY</ns2:pubkey>
+ </ns2:identity>
+
+ </okExiting>
+ </soapenv:Body>
+</soapenv:Envelope>
+
View
141 ctx-agent/ctx/lib/pylib/embeddedET/ElementInclude.py
@@ -0,0 +1,141 @@
+#
+# ElementTree
+# $Id: ElementInclude.py,v 1.1.1.1 2008/11/11 20:27:50 tfreeman Exp $
+#
+# limited xinclude support for element trees
+#
+# history:
+# 2003-08-15 fl created
+# 2003-11-14 fl fixed default loader
+#
+# Copyright (c) 2003-2004 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2004 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+##
+# Limited XInclude support for the ElementTree package.
+##
+
+import copy
+import ElementTree
+
+XINCLUDE = "{http://www.w3.org/2001/XInclude}"
+
+XINCLUDE_INCLUDE = XINCLUDE + "include"
+XINCLUDE_FALLBACK = XINCLUDE + "fallback"
+
+##
+# Fatal include error.
+
+class FatalIncludeError(SyntaxError):
+ pass
+
+##
+# Default loader. This loader reads an included resource from disk.
+#
+# @param href Resource reference.
+# @param parse Parse mode. Either "xml" or "text".
+# @param encoding Optional text encoding.
+# @return The expanded resource. If the parse mode is "xml", this
+# is an ElementTree instance. If the parse mode is "text", this
+# is a Unicode string. If the loader fails, it can return None
+# or raise an IOError exception.
+# @throws IOError If the loader fails to load the resource.
+
+def default_loader(href, parse, encoding=None):
+ file = open(href)
+ if parse == "xml":
+ data = ElementTree.parse(file).getroot()
+ else:
+ data = file.read()
+ if encoding:
+ data = data.decode(encoding)
+ file.close()
+ return data
+
+##
+# Expand XInclude directives.
+#
+# @param elem Root element.
+# @param loader Optional resource loader. If omitted, it defaults
+# to {@link default_loader}. If given, it should be a callable
+# that implements the same interface as <b>default_loader</b>.
+# @throws FatalIncludeError If the function fails to include a given
+# resource, or if the tree contains malformed XInclude elements.
+# @throws IOError If the function fails to load a given resource.
+
+def include(elem, loader=None):
+ if loader is None:
+ loader = default_loader
+ # look for xinclude elements
+ i = 0
+ while i < len(elem):
+ e = elem[i]
+ if e.tag == XINCLUDE_INCLUDE:
+ # process xinclude directive
+ href = e.get("href")
+ parse = e.get("parse", "xml")
+ if parse == "xml":
+ node = loader(href, parse)
+ if node is None:
+ raise FatalIncludeError(
+ "cannot load %r as %r" % (href, parse)
+ )
+ node = copy.copy(node)
+ if e.tail:
+ node.tail = (node.tail or "") + e.tail
+ elem[i] = node
+ elif parse == "text":
+ text = loader(href, parse, e.get("encoding"))
+ if text is None:
+ raise FatalIncludeError(
+ "cannot load %r as %r" % (href, parse)
+ )
+ if i:
+ node = elem[i-1]
+ node.tail = (node.tail or "") + text
+ else:
+ elem.text = (elem.text or "") + text + (e.tail or "")
+ del elem[i]
+ continue
+ else:
+ raise FatalIncludeError(
+ "unknown parse type in xi:include tag (%r)" % parse
+ )
+ elif e.tag == XINCLUDE_FALLBACK:
+ raise FatalIncludeError(
+ "xi:fallback tag must be child of xi:include (%r)" % e.tag
+ )
+ else:
+ include(e, loader)
+ i = i + 1
+
View
196 ctx-agent/ctx/lib/pylib/embeddedET/ElementPath.py
@@ -0,0 +1,196 @@
+#
+# ElementTree
+# $Id: ElementPath.py,v 1.1.1.1 2008/11/11 20:27:50 tfreeman Exp $
+#
+# limited xpath support for element trees
+#
+# history:
+# 2003-05-23 fl created
+# 2003-05-28 fl added support for // etc
+# 2003-08-27 fl fixed parsing of periods in element names
+#
+# Copyright (c) 2003-2004 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2004 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+##
+# Implementation module for XPath support. There's usually no reason
+# to import this module directly; the <b>ElementTree</b> does this for
+# you, if needed.
+##
+
+import re
+
+xpath_tokenizer = re.compile(
+ "(::|\.\.|\(\)|[/.*:\[\]\(\)@=])|((?:\{[^}]+\})?[^/:\[\]\(\)@=\s]+)|\s+"
+ ).findall
+
+class xpath_descendant_or_self:
+ pass
+
+##
+# Wrapper for a compiled XPath.
+
+class Path:
+
+ ##
+ # Create an Path instance from an XPath expression.
+
+ def __init__(self, path):
+ tokens = xpath_tokenizer(path)
+ # the current version supports 'path/path'-style expressions only
+ self.path = []
+ self.tag = None
+ if tokens and tokens[0][0] == "/":
+ raise SyntaxError("cannot use absolute path on element")
+ while tokens:
+ op, tag = tokens.pop(0)
+ if tag or op == "*":
+ self.path.append(tag or op)
+ elif op == ".":
+ pass
+ elif op == "/":
+ self.path.append(xpath_descendant_or_self())
+ continue
+ else:
+ raise SyntaxError("unsupported path syntax (%s)" % op)
+ if tokens:
+ op, tag = tokens.pop(0)
+ if op != "/":
+ raise SyntaxError(
+ "expected path separator (%s)" % (op or tag)
+ )
+ if self.path and isinstance(self.path[-1], xpath_descendant_or_self):
+ raise SyntaxError("path cannot end with //")
+ if len(self.path) == 1 and isinstance(self.path[0], type("")):
+ self.tag = self.path[0]
+
+ ##
+ # Find first matching object.
+
+ def find(self, element):
+ tag = self.tag
+ if tag is None:
+ nodeset = self.findall(element)
+ if not nodeset:
+ return None
+ return nodeset[0]
+ for elem in element:
+ if elem.tag == tag:
+ return elem
+ return None
+
+ ##
+ # Find text for first matching object.
+
+ def findtext(self, element, default=None):
+ tag = self.tag
+ if tag is None:
+ nodeset = self.findall(element)
+ if not nodeset:
+ return default
+ return nodeset[0].text or ""
+ for elem in element:
+ if elem.tag == tag:
+ return elem.text or ""
+ return default
+
+ ##
+ # Find all matching objects.
+
+ def findall(self, element):
+ nodeset = [element]
+ index = 0
+ while 1:
+ try:
+ path = self.path[index]
+ index = index + 1
+ except IndexError:
+ return nodeset
+ set = []
+ if isinstance(path, xpath_descendant_or_self):
+ try:
+ tag = self.path[index]
+ if not isinstance(tag, type("")):
+ tag = None
+ else:
+ index = index + 1
+ except IndexError:
+ tag = None # invalid path
+ for node in nodeset:
+ new = list(node.getiterator(tag))
+ if new and new[0] is node:
+ set.extend(new[1:])
+ else:
+ set.extend(new)
+ else:
+ for node in nodeset:
+ for node in node:
+ if path == "*" or node.tag == path:
+ set.append(node)
+ if not set:
+ return []
+ nodeset = set
+
+_cache = {}
+
+##
+# (Internal) Compile path.
+
+def _compile(path):
+ p = _cache.get(path)
+ if p is not None:
+ return p
+ p = Path(path)
+ if len(_cache) >= 100:
+ _cache.clear()
+ _cache[path] = p
+ return p
+
+##
+# Find first matching object.
+
+def find(element, path):
+ return _compile(path).find(element)
+
+##
+# Find text for first matching object.
+
+def findtext(element, path, default=None):
+ return _compile(path).findtext(element, default)
+
+##
+# Find all matching objects.
+
+def findall(element, path):
+ return _compile(path).findall(element)
+
View
1,254 ctx-agent/ctx/lib/pylib/embeddedET/ElementTree.py
@@ -0,0 +1,1254 @@
+#
+# ElementTree
+# $Id: ElementTree.py,v 1.1.1.1 2008/11/11 20:27:50 tfreeman Exp $
+#
+# light-weight XML support for Python 1.5.2 and later.
+#
+# history:
+# 2001-10-20 fl created (from various sources)
+# 2001-11-01 fl return root from parse method
+# 2002-02-16 fl sort attributes in lexical order
+# 2002-04-06 fl TreeBuilder refactoring, added PythonDoc markup
+# 2002-05-01 fl finished TreeBuilder refactoring
+# 2002-07-14 fl added basic namespace support to ElementTree.write
+# 2002-07-25 fl added QName attribute support
+# 2002-10-20 fl fixed encoding in write
+# 2002-11-24 fl changed default encoding to ascii; fixed attribute encoding
+# 2002-11-27 fl accept file objects or file names for parse/write
+# 2002-12-04 fl moved XMLTreeBuilder back to this module
+# 2003-01-11 fl fixed entity encoding glitch for us-ascii
+# 2003-02-13 fl added XML literal factory
+# 2003-02-21 fl added ProcessingInstruction/PI factory
+# 2003-05-11 fl added tostring/fromstring helpers
+# 2003-05-26 fl added ElementPath support
+# 2003-07-05 fl added makeelement factory method
+# 2003-07-28 fl added more well-known namespace prefixes
+# 2003-08-15 fl fixed typo in ElementTree.findtext (Thomas Dartsch)
+# 2003-09-04 fl fall back on emulator if ElementPath is not installed
+# 2003-10-31 fl markup updates
+# 2003-11-15 fl fixed nested namespace bug
+# 2004-03-28 fl added XMLID helper
+# 2004-06-02 fl added default support to findtext
+# 2004-06-08 fl fixed encoding of non-ascii element/attribute names
+# 2004-08-23 fl take advantage of post-2.1 expat features
+# 2005-02-01 fl added iterparse implementation
+# 2005-03-02 fl fixed iterparse support for pre-2.2 versions
+#
+# Copyright (c) 1999-2005 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2005 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+__all__ = [
+ # public symbols
+ "Comment",
+ "dump",
+ "Element", "ElementTree",
+ "fromstring",
+ "iselement", "iterparse",
+ "parse",
+ "PI", "ProcessingInstruction",
+ "QName",
+ "SubElement",
+ "tostring",
+ "TreeBuilder",
+ "VERSION", "XML",
+ "XMLTreeBuilder",
+ ]
+
+##
+# The <b>Element</b> type is a flexible container object, designed to
+# store hierarchical data structures in memory. The type can be
+# described as a cross between a list and a dictionary.
+# <p>
+# Each element has a number of properties associated with it:
+# <ul>
+# <li>a <i>tag</i>. This is a string identifying what kind of data
+# this element represents (the element type, in other words).</li>
+# <li>a number of <i>attributes</i>, stored in a Python dictionary.</li>
+# <li>a <i>text</i> string.</li>
+# <li>an optional <i>tail</i> string.</li>
+# <li>a number of <i>child elements</i>, stored in a Python sequence</li>
+# </ul>
+#
+# To create an element instance, use the {@link #Element} or {@link
+# #SubElement} factory functions.
+# <p>
+# The {@link #ElementTree} class can be used to wrap an element
+# structure, and convert it from and to XML.
+##
+
+import string, sys, re
+
+class _SimpleElementPath:
+ # emulate pre-1.2 find/findtext/findall behaviour
+ def find(self, element, tag):
+ for elem in element:
+ if elem.tag == tag:
+ return elem
+ return None
+ def findtext(self, element, tag, default=None):
+ for elem in element:
+ if elem.tag == tag:
+ return elem.text or ""
+ return default
+ def findall(self, element, tag):
+ if tag[:3] == ".//":
+ return element.getiterator(tag[3:])
+ result = []
+ for elem in element:
+ if elem.tag == tag:
+ result.append(elem)
+ return result
+
+try:
+ import ElementPath
+except ImportError:
+ # FIXME: issue warning in this case?
+ ElementPath = _SimpleElementPath()
+
+# TODO: add support for custom namespace resolvers/default namespaces
+# TODO: add improved support for incremental parsing
+
+VERSION = "1.2.6"
+
+##
+# Internal element class. This class defines the Element interface,
+# and provides a reference implementation of this interface.
+# <p>
+# You should not create instances of this class directly. Use the
+# appropriate factory functions instead, such as {@link #Element}
+# and {@link #SubElement}.
+#
+# @see Element
+# @see SubElement
+# @see Comment
+# @see ProcessingInstruction
+
+class _ElementInterface:
+ # <tag attrib>text<child/>...</tag>tail
+
+ ##
+ # (Attribute) Element tag.
+
+ tag = None
+
+ ##
+ # (Attribute) Element attribute dictionary. Where possible, use
+ # {@link #_ElementInterface.get},
+ # {@link #_ElementInterface.set},
+ # {@link #_ElementInterface.keys}, and
+ # {@link #_ElementInterface.items} to access
+ # element attributes.
+
+ attrib = None
+
+ ##
+ # (Attribute) Text before first subelement. This is either a
+ # string or the value None, if there was no text.
+
+ text = None
+
+ ##
+ # (Attribute) Text after this element's end tag, but before the
+ # next sibling element's start tag. This is either a string or
+ # the value None, if there was no text.
+
+ tail = None # text after end tag, if any
+
+ def __init__(self, tag, attrib):
+ self.tag = tag
+ self.attrib = attrib
+ self._children = []
+
+ def __repr__(self):
+ return "<Element %s at %x>" % (self.tag, id(self))
+
+ ##
+ # Creates a new element object of the same type as this element.
+ #
+ # @param tag Element tag.
+ # @param attrib Element attributes, given as a dictionary.
+ # @return A new element instance.
+
+ def makeelement(self, tag, attrib):
+ return Element(tag, attrib)
+
+ ##
+ # Returns the number of subelements.
+ #
+ # @return The number of subelements.
+
+ def __len__(self):
+ return len(self._children)
+
+ ##
+ # Returns the given subelement.
+ #
+ # @param index What subelement to return.
+ # @return The given subelement.
+ # @exception IndexError If the given element does not exist.
+
+ def __getitem__(self, index):
+ return self._children[index]
+
+ ##
+ # Replaces the given subelement.
+ #
+ # @param index What subelement to replace.
+ # @param element The new element value.
+ # @exception IndexError If the given element does not exist.
+ # @exception AssertionError If element is not a valid object.
+
+ def __setitem__(self, index, element):
+ assert iselement(element)
+ self._children[index] = element
+
+ ##
+ # Deletes the given subelement.
+ #
+ # @param index What subelement to delete.
+ # @exception IndexError If the given element does not exist.
+
+ def __delitem__(self, index):
+ del self._children[index]
+
+ ##
+ # Returns a list containing subelements in the given range.
+ #
+ # @param start The first subelement to return.
+ # @param stop The first subelement that shouldn't be returned.
+ # @return A sequence object containing subelements.
+
+ def __getslice__(self, start, stop):
+ return self._children[start:stop]
+
+ ##
+ # Replaces a number of subelements with elements from a sequence.
+ #
+ # @param start The first subelement to replace.
+ # @param stop The first subelement that shouldn't be replaced.
+ # @param elements A sequence object with zero or more elements.
+ # @exception AssertionError If a sequence member is not a valid object.
+
+ def __setslice__(self, start, stop, elements):
+ for element in elements:
+ assert iselement(element)
+ self._children[start:stop] = list(elements)
+
+ ##
+ # Deletes a number of subelements.
+ #
+ # @param start The first subelement to delete.
+ # @param stop The first subelement to leave in there.
+
+ def __delslice__(self, start, stop):
+ del self._children[start:stop]
+
+ ##
+ # Adds a subelement to the end of this element.
+ #
+ # @param element The element to add.
+ # @exception AssertionError If a sequence member is not a valid object.
+
+ def append(self, element):
+ assert iselement(element)
+ self._children.append(element)
+
+ ##
+ # Inserts a subelement at the given position in this element.
+ #
+ # @param index Where to insert the new subelement.
+ # @exception AssertionError If the element is not a valid object.
+
+ def insert(self, index, element):
+ assert iselement(element)
+ self._children.insert(index, element)
+
+ ##
+ # Removes a matching subelement. Unlike the <b>find</b> methods,
+ # this method compares elements based on identity, not on tag
+ # value or contents.
+ #
+ # @param element What element to remove.
+ # @exception ValueError If a matching element could not be found.
+ # @exception AssertionError If the element is not a valid object.
+
+ def remove(self, element):
+ assert iselement(element)
+ self._children.remove(element)
+
+ ##
+ # Returns all subelements. The elements are returned in document
+ # order.
+ #
+ # @return A list of subelements.
+ # @defreturn list of Element instances
+
+ def getchildren(self):
+ return self._children
+
+ ##
+ # Finds the first matching subelement, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @return The first matching element, or None if no element was found.
+ # @defreturn Element or None
+
+ def find(self, path):
+ return ElementPath.find(self, path)
+
+ ##
+ # Finds text for the first matching subelement, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @param default What to return if the element was not found.
+ # @return The text content of the first matching element, or the
+ # default value no element was found. Note that if the element
+ # has is found, but has no text content, this method returns an
+ # empty string.
+ # @defreturn string
+
+ def findtext(self, path, default=None):
+ return ElementPath.findtext(self, path, default)
+
+ ##
+ # Finds all matching subelements, by tag name or path.
+ #
+ # @param path What element to look for.
+ # @return A list or iterator containing all matching elements,
+ # in document order.
+ # @defreturn list of Element instances
+
+ def findall(self, path):
+ return ElementPath.findall(self, path)
+
+ ##
+ # Resets an element. This function removes all subelements, clears
+ # all attributes, and sets the text and tail attributes to None.
+
+ def clear(self):
+ self.attrib.clear()
+ self._children = []
+ self.text = self.tail = None
+
+ ##
+ # Gets an element attribute.
+ #
+ # @param key What attribute to look for.
+ # @param default What to return if the attribute was not found.
+ # @return The attribute value, or the default value, if the
+ # attribute was not found.
+ # @defreturn string or None
+
+ def get(self, key, default=None):
+ return self.attrib.get(key, default)
+
+ ##
+ # Sets an element attribute.
+ #
+ # @param key What attribute to set.
+ # @param value The attribute value.
+
+ def set(self, key, value):
+ self.attrib[key] = value
+
+ ##
+ # Gets a list of attribute names. The names are returned in an
+ # arbitrary order (just like for an ordinary Python dictionary).
+ #
+ # @return A list of element attribute names.
+ # @defreturn list of strings
+
+ def keys(self):
+ return self.attrib.keys()
+
+ ##
+ # Gets element attributes, as a sequence. The attributes are
+ # returned in an arbitrary order.
+ #
+ # @return A list of (name, value) tuples for all attributes.
+ # @defreturn list of (string, string) tuples
+
+ def items(self):
+ return self.attrib.items()
+
+ ##
+ # Creates a tree iterator. The iterator loops over this element
+ # and all subelements, in document order, and returns all elements
+ # with a matching tag.
+ # <p>
+ # If the tree structure is modified during iteration, the result
+ # is undefined.
+ #
+ # @param tag What tags to look for (default is to return all elements).
+ # @return A list or iterator containing all the matching elements.
+ # @defreturn list or iterator
+
+ def getiterator(self, tag=None):
+ nodes = []
+ if tag == "*":
+ tag = None
+ if tag is None or self.tag == tag:
+ nodes.append(self)
+ for node in self._children:
+ nodes.extend(node.getiterator(tag))
+ return nodes
+
+# compatibility
+_Element = _ElementInterface
+
+##
+# Element factory. This function returns an object implementing the
+# standard Element interface. The exact class or type of that object
+# is implementation dependent, but it will always be compatible with
+# the {@link #_ElementInterface} class in this module.
+# <p>
+# The element name, attribute names, and attribute values can be
+# either 8-bit ASCII strings or Unicode strings.
+#
+# @param tag The element name.
+# @param attrib An optional dictionary, containing element attributes.
+# @param **extra Additional attributes, given as keyword arguments.
+# @return An element instance.
+# @defreturn Element
+
+def Element(tag, attrib={}, **extra):
+ attrib = attrib.copy()
+ attrib.update(extra)
+ return _ElementInterface(tag, attrib)
+
+##
+# Subelement factory. This function creates an element instance, and
+# appends it to an existing element.
+# <p>
+# The element name, attribute names, and attribute values can be
+# either 8-bit ASCII strings or Unicode strings.
+#
+# @param parent The parent element.
+# @param tag The subelement name.
+# @param attrib An optional dictionary, containing element attributes.
+# @param **extra Additional attributes, given as keyword arguments.
+# @return An element instance.
+# @defreturn Element
+
+def SubElement(parent, tag, attrib={}, **extra):
+ attrib = attrib.copy()
+ attrib.update(extra)
+ element = parent.makeelement(tag, attrib)
+ parent.append(element)
+ return element
+
+##
+# Comment element factory. This factory function creates a special
+# element that will be serialized as an XML comment.
+# <p>
+# The comment string can be either an 8-bit ASCII string or a Unicode
+# string.
+#
+# @param text A string containing the comment string.
+# @return An element instance, representing a comment.
+# @defreturn Element
+
+def Comment(text=None):
+ element = Element(Comment)
+ element.text = text
+ return element
+
+##
+# PI element factory. This factory function creates a special element
+# that will be serialized as an XML processing instruction.
+#
+# @param target A string containing the PI target.
+# @param text A string containing the PI contents, if any.
+# @return An element instance, representing a PI.
+# @defreturn Element
+
+def ProcessingInstruction(target, text=None):
+ element = Element(ProcessingInstruction)
+ element.text = target
+ if text:
+ element.text = element.text + " " + text
+ return element
+
+PI = ProcessingInstruction
+
+##
+# QName wrapper. This can be used to wrap a QName attribute value, in
+# order to get proper namespace handling on output.
+#
+# @param text A string containing the QName value, in the form {uri}local,
+# or, if the tag argument is given, the URI part of a QName.
+# @param tag Optional tag. If given, the first argument is interpreted as
+# an URI, and this argument is interpreted as a local name.
+# @return An opaque object, representing the QName.
+
+class QName:
+ def __init__(self, text_or_uri, tag=None):
+ if tag:
+ text_or_uri = "{%s}%s" % (text_or_uri, tag)
+ self.text = text_or_uri
+ def __str__(self):
+ return self.text
+ def __hash__(self):
+ return hash(self.text)
+ def __cmp__(self, other):
+ if isinstance(other, QName):
+ return cmp(self.text, other.text)
+ return cmp(self.text, other)
+
+##
+# ElementTree wrapper class. This class represents an entire element
+# hierarchy, and adds some extra support for serialization to and from
+# standard XML.
+#
+# @param element Optional root element.
+# @keyparam file Optional file handle or name. If given, the
+# tree is initialized with the contents of this XML file.
+
+class ElementTree:
+
+ def __init__(self, element=None, file=None):
+ assert element is None or iselement(element)
+ self._root = element # first node
+ if file:
+ self.parse(file)
+
+ ##
+ # Gets the root element for this tree.
+ #
+ # @return An element instance.
+ # @defreturn Element
+
+ def getroot(self):
+ return self._root
+
+ ##
+ # Replaces the root element for this tree. This discards the
+ # current contents of the tree, and replaces it with the given
+ # element. Use with care.
+ #
+ # @param element An element instance.
+
+ def _setroot(self, element):
+ assert iselement(element)
+ self._root = element
+
+ ##
+ # Loads an external XML document into this element tree.
+ #
+ # @param source A file name or file object.
+ # @param parser An optional parser instance. If not given, the
+ # standard {@link XMLTreeBuilder} parser is used.
+ # @return The document root element.
+ # @defreturn Element
+
+ def parse(self, source, parser=None):
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ if not parser:
+ parser = XMLTreeBuilder()
+ while 1:
+ data = source.read(32768)
+ if not data:
+ break
+ parser.feed(data)
+ self._root = parser.close()
+ return self._root
+
+ ##
+ # Creates a tree iterator for the root element. The iterator loops
+ # over all elements in this tree, in document order.
+ #
+ # @param tag What tags to look for (default is to return all elements)
+ # @return An iterator.
+ # @defreturn iterator
+
+ def getiterator(self, tag=None):
+ assert self._root is not None
+ return self._root.getiterator(tag)
+
+ ##
+ # Finds the first toplevel element with given tag.
+ # Same as getroot().find(path).
+ #
+ # @param path What element to look for.
+ # @return The first matching element, or None if no element was found.
+ # @defreturn Element or None
+
+ def find(self, path):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.find(path)
+
+ ##
+ # Finds the element text for the first toplevel element with given
+ # tag. Same as getroot().findtext(path).
+ #
+ # @param path What toplevel element to look for.
+ # @param default What to return if the element was not found.
+ # @return The text content of the first matching element, or the
+ # default value no element was found. Note that if the element
+ # has is found, but has no text content, this method returns an
+ # empty string.
+ # @defreturn string
+
+ def findtext(self, path, default=None):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.findtext(path, default)
+
+ ##
+ # Finds all toplevel elements with the given tag.
+ # Same as getroot().findall(path).
+ #
+ # @param path What element to look for.
+ # @return A list or iterator containing all matching elements,
+ # in document order.
+ # @defreturn list of Element instances
+
+ def findall(self, path):
+ assert self._root is not None
+ if path[:1] == "/":
+ path = "." + path
+ return self._root.findall(path)
+
+ ##
+ # Writes the element tree to a file, as XML.
+ #
+ # @param file A file name, or a file object opened for writing.
+ # @param encoding Optional output encoding (default is US-ASCII).
+
+ def write(self, file, encoding="us-ascii"):
+ assert self._root is not None
+ if not hasattr(file, "write"):
+ file = open(file, "wb")
+ if not encoding:
+ encoding = "us-ascii"
+ elif encoding != "utf-8" and encoding != "us-ascii":
+ file.write("<?xml version='1.0' encoding='%s'?>\n" % encoding)
+ self._write(file, self._root, encoding, {})
+
+ def _write(self, file, node, encoding, namespaces):
+ # write XML to file
+ tag = node.tag
+ if tag is Comment:
+ file.write("<!-- %s -->" % _escape_cdata(node.text, encoding))
+ elif tag is ProcessingInstruction:
+ file.write("<?%s?>" % _escape_cdata(node.text, encoding))
+ else:
+ items = node.items()
+ xmlns_items = [] # new namespaces in this scope
+ try:
+ if isinstance(tag, QName) or tag[:1] == "{":
+ tag, xmlns = fixtag(tag, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(tag)
+ file.write("<" + _encode(tag, encoding))
+ if items or xmlns_items:
+ items.sort() # lexical order
+ for k, v in items:
+ try:
+ if isinstance(k, QName) or k[:1] == "{":
+ k, xmlns = fixtag(k, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(k)
+ try:
+ if isinstance(v, QName):
+ v, xmlns = fixtag(v, namespaces)
+ if xmlns: xmlns_items.append(xmlns)
+ except TypeError:
+ _raise_serialization_error(v)
+ file.write(" %s=\"%s\"" % (_encode(k, encoding),
+ _escape_attrib(v, encoding)))
+ for k, v in xmlns_items:
+ file.write(" %s=\"%s\"" % (_encode(k, encoding),
+ _escape_attrib(v, encoding)))
+ if node.text or len(node):
+ file.write(">")
+ if node.text:
+ file.write(_escape_cdata(node.text, encoding))
+ for n in node:
+ self._write(file, n, encoding, namespaces)
+ file.write("</" + _encode(tag, encoding) + ">")
+ else:
+ file.write(" />")
+ for k, v in xmlns_items:
+ del namespaces[v]
+ if node.tail:
+ file.write(_escape_cdata(node.tail, encoding))
+
+# --------------------------------------------------------------------
+# helpers
+
+##
+# Checks if an object appears to be a valid element object.
+#
+# @param An element instance.
+# @return A true value if this is an element object.
+# @defreturn flag
+
+def iselement(element):
+ # FIXME: not sure about this; might be a better idea to look
+ # for tag/attrib/text attributes
+ return isinstance(element, _ElementInterface) or hasattr(element, "tag")
+
+##
+# Writes an element tree or element structure to sys.stdout. This
+# function should be used for debugging only.
+# <p>
+# The exact output format is implementation dependent. In this
+# version, it's written as an ordinary XML file.
+#
+# @param elem An element tree or an individual element.
+
+def dump(elem):
+ # debugging
+ if not isinstance(elem, ElementTree):
+ elem = ElementTree(elem)
+ elem.write(sys.stdout)
+ tail = elem.getroot().tail
+ if not tail or tail[-1] != "\n":
+ sys.stdout.write("\n")
+
+def _encode(s, encoding):
+ try:
+ return s.encode(encoding)
+ except AttributeError:
+ return s # 1.5.2: assume the string uses the right encoding
+
+if sys.version[:3] == "1.5":
+ _escape = re.compile(r"[&<>\"\x80-\xff]+") # 1.5.2
+else:
+ _escape = re.compile(eval(r'u"[&<>\"\u0080-\uffff]+"'))
+
+_escape_map = {
+ "&": "&amp;",
+ "<": "&lt;",
+ ">": "&gt;",
+ '"': "&quot;",
+}
+
+_namespace_map = {
+ # "well-known" namespace prefixes
+ "http://www.w3.org/XML/1998/namespace": "xml",
+ "http://www.w3.org/1999/xhtml": "html",
+ "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
+ "http://schemas.xmlsoap.org/wsdl/": "wsdl",
+}
+
+def _raise_serialization_error(text):
+ raise TypeError(
+ "cannot serialize %r (type %s)" % (text, type(text).__name__)
+ )
+
+def _encode_entity(text, pattern=_escape):
+ # map reserved and non-ascii characters to numerical entities
+ def escape_entities(m, map=_escape_map):
+ out = []
+ append = out.append
+ for char in m.group():
+ text = map.get(char)
+ if text is None:
+ text = "&#%d;" % ord(char)
+ append(text)
+ return string.join(out, "")
+ try:
+ return _encode(pattern.sub(escape_entities, text), "ascii")
+ except TypeError:
+ _raise_serialization_error(text)
+
+#
+# the following functions assume an ascii-compatible encoding
+# (or "utf-16")
+
+def _escape_cdata(text, encoding=None, replace=string.replace):
+ # escape character data
+ try:
+ if encoding:
+ try:
+ text = _encode(text, encoding)
+ except UnicodeError:
+ return _encode_entity(text)
+ text = replace(text, "&", "&amp;")
+ text = replace(text, "<", "&lt;")
+ text = replace(text, ">", "&gt;")
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_attrib(text, encoding=None, replace=string.replace):
+ # escape attribute value
+ try:
+ if encoding:
+ try:
+ text = _encode(text, encoding)
+ except UnicodeError:
+ return _encode_entity(text)
+ text = replace(text, "&", "&amp;")
+ text = replace(text, "'", "&apos;") # FIXME: overkill
+ text = replace(text, "\"", "&quot;")
+ text = replace(text, "<", "&lt;")
+ text = replace(text, ">", "&gt;")
+ return text
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def fixtag(tag, namespaces):
+ # given a decorated tag (of the form {uri}tag), return prefixed
+ # tag and namespace declaration, if any
+ if isinstance(tag, QName):
+ tag = tag.text
+ namespace_uri, tag = string.split(tag[1:], "}", 1)
+ prefix = namespaces.get(namespace_uri)
+ if prefix is None:
+ prefix = _namespace_map.get(namespace_uri)
+ if prefix is None:
+ prefix = "ns%d" % len(namespaces)
+ namespaces[namespace_uri] = prefix
+ if prefix == "xml":
+ xmlns = None
+ else:
+ xmlns = ("xmlns:%s" % prefix, namespace_uri)
+ else:
+ xmlns = None
+ return "%s:%s" % (prefix, tag), xmlns
+
+##
+# Parses an XML document into an element tree.
+#
+# @param source A filename or file object containing XML data.
+# @param parser An optional parser instance. If not given, the
+# standard {@link XMLTreeBuilder} parser is used.
+# @return An ElementTree instance
+
+def parse(source, parser=None):
+ tree = ElementTree()
+ tree.parse(source, parser)
+ return tree
+
+##
+# Parses an XML document into an element tree incrementally, and reports
+# what's going on to the user.
+#
+# @param source A filename or file object containing XML data.
+# @param events A list of events to report back. If omitted, only "end"
+# events are reported.
+# @return A (event, elem) iterator.
+
+class iterparse:
+
+ def __init__(self, source, events=None):
+ if not hasattr(source, "read"):
+ source = open(source, "rb")
+ self._file = source
+ self._events = []
+ self._index = 0
+ self.root = self._root = None
+ self._parser = XMLTreeBuilder()
+ # wire up the parser for event reporting
+ parser = self._parser._parser
+ append = self._events.append
+ if events is None:
+ events = ["end"]
+ for event in events:
+ if event == "start":
+ try:
+ parser.ordered_attributes = 1
+ parser.specified_attributes = 1
+ def handler(tag, attrib_in, event=event, append=append,
+ start=self._parser._start_list):
+ append((event, start(tag, attrib_in)))
+ parser.StartElementHandler = handler
+ except AttributeError:
+ def handler(tag, attrib_in, event=event, append=append,
+ start=self._parser._start):
+ append((event, start(tag, attrib_in)))
+ parser.StartElementHandler = handler
+ elif event == "end":
+ def handler(tag, event=event, append=append,
+ end=self._parser._end):
+ append((event, end(tag)))
+ parser.EndElementHandler = handler
+ elif event == "start-ns":
+ def handler(prefix, uri, event=event, append=append):
+ try:
+ uri = _encode(uri, "ascii")
+ except UnicodeError:
+ pass
+ append((event, (prefix or "", uri)))
+ parser.StartNamespaceDeclHandler = handler
+ elif event == "end-ns":
+ def handler(prefix, event=event, append=append):
+ append((event, None))
+ parser.EndNamespaceDeclHandler = handler
+