Skip to content
Permalink
Browse files

Merge from trunk.

  • Loading branch information...
cyrusdaboo committed Jul 7, 2016
2 parents 07c6d45 + c6ee47d commit ac8961121988a292c1fd5b2793d86cc62fd444dc
@@ -602,7 +602,7 @@ c_dependencies () {
if command -v postgres > /dev/null; then
using_system "Postgres";
else
local v="9.3.10";
local v="9.5.3";
local n="postgresql";
local p="${n}-${v}";

@@ -612,10 +612,10 @@ c_dependencies () {
local enable_dtrace="";
fi;

c_dependency -m "ec2365548d08f69c8023eddd4f2d1a28" \
c_dependency -m "3f0c388566c688c82b01a0edf1e6b7a0" \
"PostgreSQL" "${p}" \
"http://ftp.postgresql.org/pub/source/v${v}/${p}.tar.bz2" \
--with-python ${enable_dtrace};
${enable_dtrace};
fi;

}
@@ -678,6 +678,12 @@ py_dependencies () {
--no-setuptools \
${virtualenv_opts} \
"${py_virtualenv}";
case "$(uname -s)" in
Darwin)
echo "macOS virtualenv codesign fix."
cp "/usr/bin/python" "${py_bindir}/python";
;;
esac;
fi;

cd "${wd}";
@@ -711,9 +717,11 @@ py_dependencies () {

ruler "Patching Python requirements";
echo "";
if [ ! -e "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied" ]; then
twisted_version=$("${python}" -c 'from twisted._version import version; print version.base()');
if [ ! -e "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied.${twisted_version}" ]; then
apply_patches "Twisted" "${py_virtualenv}/lib/python2.7/site-packages"
touch "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied";
find "${py_virtualenv}/lib/python2.7/site-packages/twisted" -type f -name '.patch_applied*' -print0 | xargs -0 rm -f;
touch "${py_virtualenv}/lib/python2.7/site-packages/twisted/.patch_applied.${twisted_version}";
fi;

echo "";
@@ -735,37 +743,15 @@ macos_oracle () {

bootstrap_virtualenv () {
mkdir -p "${py_ve_tools}";
mkdir -p "${py_ve_tools}/lib";
mkdir -p "${py_ve_tools}/junk";
export PYTHONUSERBASE="${py_ve_tools}"

for pkg in \
setuptools-18.5 \
pip-7.1.2 \
virtualenv-13.1.2 \
setuptools==18.5 \
pip==8.1.2 \
virtualenv==15.0.2 \
; do
local name="${pkg%-*}";
local version="${pkg#*-}";
local first="$(echo "${name}" | sed 's|^\(.\).*$|\1|')";
local url="https://pypi.python.org/packages/source/${first}/${name}/${pkg}.tar.gz";

ruler "Downloading ${pkg}";

local tmp="$(mktemp -d -t ccsXXXXX)";

curl -L "${url}" | tar -C "${tmp}" -xvzf -;

cd "${tmp}/$(basename "${pkg}")";
PYTHONPATH="${py_ve_tools}/lib" \
"${bootstrap_python}" setup.py install \
--install-base="${py_ve_tools}" \
--install-lib="${py_ve_tools}/lib" \
--install-headers="${py_ve_tools}/junk" \
--install-scripts="${py_ve_tools}/junk" \
--install-data="${py_ve_tools}/junk" \
; \
cd "${wd}";

rm -rf "${tmp}";
ruler "Installing ${pkg}";
"${bootstrap_python}" -m pip install -I --user "${pkg}";
done;
}

@@ -776,7 +762,7 @@ pip_download () {
"${python}" -m pip install \
--disable-pip-version-check \
--download="${dev_home}/pip_downloads" \
--pre --allow-all-external \
--pre \
--no-cache-dir \
--log-file="${dev_home}/pip.log" \
"$@";
@@ -786,7 +772,7 @@ pip_download () {
pip_install_from_cache () {
"${python}" -m pip install \
--disable-pip-version-check \
--pre --allow-all-external \
--pre \
--no-index \
--no-cache-dir \
--find-links="${dev_patches}" \
@@ -799,7 +785,7 @@ pip_install_from_cache () {
pip_download_and_install () {
"${python}" -m pip install \
--disable-pip-version-check \
--pre --allow-all-external \
--pre \
--no-cache-dir \
--find-links="${dev_patches}" \
--log-file="${dev_home}/pip.log" \
@@ -33,8 +33,8 @@ sim="${wd}/contrib/performance/loadtest/sim.py";
##

verbose="";
config="${wd}/contrib/performance/loadtest/config.plist";
clients="${wd}/contrib/performance/loadtest/clients.plist";
config="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist";
clients="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist";
runtime="--runtime 300";
logfile="";

@@ -33,8 +33,8 @@ sim="${wd}/contrib/performance/loadtest/sim.py";
##

verbose="";
config="${wd}/contrib/performance/loadtest/config.plist";
clients="${wd}/contrib/performance/loadtest/clients.plist";
config="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-config.plist";
clients="${wd}/contrib/performance/loadtest/standard-configs/accelerated-activity-clients.plist";
runtime="--runtime 300";
logfile="";

@@ -66,8 +66,7 @@
from twext.internet.ssl import ChainingOpenSSLContextFactory
from twext.internet.tcp import MaxAcceptTCPServer, MaxAcceptSSLServer
from twext.enterprise.adbapi2 import ConnectionPool
from twext.enterprise.ienterprise import ORACLE_DIALECT
from twext.enterprise.ienterprise import POSTGRES_DIALECT
from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
from twext.enterprise.jobs.queue import NonPerformingQueuer
from twext.enterprise.jobs.queue import ControllerQueue
from twext.enterprise.jobs.queue import WorkerFactory as QueueWorkerFactory
@@ -1565,7 +1564,7 @@ def storageService(
@rtype: L{IService}
"""

def createSubServiceFactory(dbtype):
def createSubServiceFactory(dbtype, dbfeatures=()):
if dbtype == "":
dialect = POSTGRES_DIALECT
paramstyle = "pyformat"
@@ -1579,8 +1578,8 @@ def createSubServiceFactory(dbtype):
def subServiceFactory(connectionFactory, storageService):
ms = MultiService()
cp = ConnectionPool(
connectionFactory, dialect=dialect,
paramstyle=paramstyle,
connectionFactory,
dbtype=DatabaseType(dialect, paramstyle, dbfeatures),
maxConnections=config.MaxDBConnectionsPerPool
)
cp.setServiceParent(ms)
@@ -1680,13 +1679,13 @@ def subServiceFactory(connectionFactory, storageService):
# to it.
pgserv = pgServiceFromConfig(
config,
createSubServiceFactory(""),
createSubServiceFactory("", config.DBFeatures),
uid=overrideUID, gid=overrideGID
)
return pgserv
else:
# Connect to a database that is already running.
return createSubServiceFactory(config.DBType)(
return createSubServiceFactory(config.DBType, config.DBFeatures)(
DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect, None
)
else:
@@ -44,8 +44,7 @@

from twext.enterprise.adbapi2 import ConnectionPool, ConnectionPoolConnection
from twext.enterprise.adbapi2 import ConnectionPoolClient
from twext.enterprise.ienterprise import ORACLE_DIALECT
from twext.enterprise.ienterprise import POSTGRES_DIALECT
from twext.enterprise.ienterprise import POSTGRES_DIALECT, ORACLE_DIALECT, DatabaseType
from twext.internet.ssl import ChainingOpenSSLContextFactory
from twext.python.filepath import CachingFilePath
from twext.python.filepath import CachingFilePath as FilePath
@@ -179,14 +178,14 @@ def getHost(self):



def transactionFactoryFromFD(dbampfd, dialect, paramstyle):
def transactionFactoryFromFD(dbampfd, dbtype):
"""
Create a transaction factory from an inherited file descriptor, such as one
created by L{ConnectionDispenser}.
"""
skt = fromfd(dbampfd, AF_UNIX, SOCK_STREAM)
os.close(dbampfd)
protocol = ConnectionPoolClient(dialect=dialect, paramstyle=paramstyle)
protocol = ConnectionPoolClient(dbtype=dbtype)
transport = ConnectionWithPeer(skt, protocol)
protocol.makeConnection(transport)
transport.startReading()
@@ -830,7 +829,7 @@ def getDBPool(config):
pool = None
if config.DBAMPFD:
txnFactory = transactionFactoryFromFD(
int(config.DBAMPFD), dialect, paramstyle
int(config.DBAMPFD), DatabaseType(dialect, paramstyle, config.DBFeatures)
)
elif not config.UseDatabase:
txnFactory = None
@@ -843,8 +842,7 @@ def getDBPool(config):
else:
connectionFactory = DBAPIConnector.connectorFor(config.DBType, **config.DatabaseConnection).connect

pool = ConnectionPool(connectionFactory, dialect=dialect,
paramstyle=paramstyle,
pool = ConnectionPool(connectionFactory, dbtype=DatabaseType(dialect, paramstyle, config.DBFeatures),
maxConnections=config.MaxDBConnectionsPerPool)
txnFactory = pool.connection
else:
@@ -108,6 +108,11 @@
<!-- Database connection -->
<key>DBType</key>
<string></string>

<!-- No features - when macOS supports postgres 9.5 and up change
this to include 'skip-locked'. -->
<key>DBFeatures</key>
<array></array>

<key>Postgres</key>
<dict>
@@ -118,35 +118,43 @@
<array>
</array>

<!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
<key>UseMetaFD</key>
<true/>

<!-- Inherited file descriptor to call recvmsg() on to receive sockets (none =
don't inherit) -->
<key>MetaFD</key>
<integer>0</integer>

<!-- Use a 'meta' FD, i.e. an FD to transmit other FDs to slave processes. -->
<key>UseMetaFD</key>
<true/>
<!-- Database configuration information.
<!-- True: database; False: files -->
<key>UseDatabase</key>
<true/>
Defines what kind of database to use: file (deprecated) or SQL. File-based
DB is only supported for migration purposes - it cannot be used for a real
service.
<!-- Timeout transactions that take longer than the specified number of
seconds. Zero means no timeouts. 5 minute default. -->
<key>TransactionTimeoutSeconds</key>
<integer>300</integer>
For an SQL-based DB, configuration of connection parameters and various
timeouts is provided. -->

<!-- When a transactions times out tell HTTP clients clients to retry after
this amount of time -->
<key>TransactionHTTPRetrySeconds</key>
<integer>300</integer>
<!-- True: database; False: files (deprecated) -->
<key>UseDatabase</key>
<true/>

<!-- 2 possible values: empty, meaning 'spawn postgres yourself', or
'postgres', meaning 'connect to a postgres database as specified by the
'DSN' configuration key. Will support more values in the future. -->
<!-- Possible values: empty, meaning 'spawn postgres yourself', or 'postgres'
or 'oracle', meaning 'connect to a postgres or Oracle database as
specified by the 'DSN' configuration key. -->
<key>DBType</key>
<string></string>

<!-- Features supported by the database
'skip-locked': SKIP LOCKED available with SELECT (remove if using postgres
&lt; v9.5) -->
<key>DBFeatures</key>
<array>
<string>skip-locked</string>
</array>

<!-- The username to use when DBType is empty -->
<key>SpawnedDBUser</key>
<string>caldav</string>
@@ -175,17 +183,17 @@
<false/>
</dict>

<!-- Use a shared database connection pool in the master process, rather than
having each client make its connections directly. -->
<key>SharedConnectionPool</key>
<false/>

<!-- Internally used by database to tell slave processes to inherit a file
descriptor and use it as an AMP connection over a UNIX socket; see
twext.enterprise.adbapi2.ConnectionPoolConnection -->
<key>DBAMPFD</key>
<integer>0</integer>

<!-- Use a shared database connection pool in the master process, rather than
having each client make its connections directly. -->
<key>SharedConnectionPool</key>
<false/>

<!-- Set to True to prevent the server or utility tools from running if the
database needs a schema upgrade. -->
<key>FailIfUpgradeNeeded</key>
@@ -203,6 +211,16 @@
<key>UpgradeHomePrefix</key>
<string></string>

<!-- Timeout transactions that take longer than the specified number of
seconds. Zero means no timeouts. 5 minute default. -->
<key>TransactionTimeoutSeconds</key>
<integer>300</integer>

<!-- When a transactions times out tell HTTP clients clients to retry after
this amount of time -->
<key>TransactionHTTPRetrySeconds</key>
<integer>300</integer>

<!-- Work queue configuration information -->

<key>WorkQueue</key>

0 comments on commit ac89611

Please sign in to comment.
You can’t perform that action at this time.