Skip to content
This repository has been archived by the owner on Feb 13, 2020. It is now read-only.

Commit

Permalink
git-svn-id: https://svn.calendarserver.org/repository/calendarserver/…
Browse files Browse the repository at this point in the history 
…CalendarServer/branches/users/cdaboo/better-next-job@15761 e27351fd-9f3e-4f54-a53b-843176b1656c
  • Loading branch information
@cyrusdaboo
cyrusdaboo committed Jul 13, 2016
1 parent b42927f commit c4498a2
Show file tree
Hide file tree
Showing 17 changed files with 362 additions and 380 deletions.
13 changes: 10 additions & 3 deletions bin/_build.sh
View file
Expand Up @@ -168,6 +168,13 @@ init_build () {
use_openssl="false" use_openssl="false"
;; ;;
esac; esac;
else
case "$(uname -s)" in
Darwin)
# Needed to build OpenSSL 64-bit on OS X
export KERNEL_BITS=64
;;
esac
fi; fi;
conditional_set requirements "${default_requirements}" conditional_set requirements "${default_requirements}"


Expand Down Expand Up @@ -490,7 +497,7 @@ c_dependencies () {
if [ ${use_openssl} == "true" ]; then if [ ${use_openssl} == "true" ]; then
ruler; ruler;


local min_ssl_version="9470095"; # OpenSSL 0.9.8zf local min_ssl_version="268443791"; # OpenSSL 1.0.2h


local ssl_version="$(c_macro openssl/ssl.h OPENSSL_VERSION_NUMBER)"; local ssl_version="$(c_macro openssl/ssl.h OPENSSL_VERSION_NUMBER)";
if [ -z "${ssl_version}" ]; then ssl_version="0x0"; fi; if [ -z "${ssl_version}" ]; then ssl_version="0x0"; fi;
Expand All @@ -499,13 +506,13 @@ c_dependencies () {
if [ "${ssl_version}" -ge "${min_ssl_version}" ]; then if [ "${ssl_version}" -ge "${min_ssl_version}" ]; then
using_system "OpenSSL"; using_system "OpenSSL";
else else
local v="0.9.8zh"; local v="1.0.2h";
local n="openssl"; local n="openssl";
local p="${n}-${v}"; local p="${n}-${v}";


# use 'config' instead of 'configure'; 'make' instead of 'jmake'. # use 'config' instead of 'configure'; 'make' instead of 'jmake'.
# also pass 'shared' to config to build shared libs. # also pass 'shared' to config to build shared libs.
c_dependency -c "config" -s "3ff71636bea85a99f4d76a10d119c09bda0421e3" \ c_dependency -c "config" -s "577585f5f5d299c44dd3c993d3c0ac7a219e4949" \
-p "make depend" -b "make" \ -p "make depend" -b "make" \
"openssl" "${p}" \ "openssl" "${p}" \
"http://www.openssl.org/source/${p}.tar.gz" "shared"; "http://www.openssl.org/source/${p}.tar.gz" "shared";
Expand Down
173 changes: 83 additions & 90 deletions calendarserver/tap/caldav.py
View file
Expand Up @@ -245,6 +245,7 @@ def __init__(self, logEnabled, logPath, logRotateLength, logMaxFiles, logRotateO
self.logRotateLength = logRotateLength self.logRotateLength = logRotateLength
self.logMaxFiles = logMaxFiles self.logMaxFiles = logMaxFiles
self.logRotateOnStart = logRotateOnStart self.logRotateOnStart = logRotateOnStart
self.name = "elms"




def setServiceParent(self, app): def setServiceParent(self, app):
Expand Down Expand Up @@ -280,7 +281,7 @@ class CalDAVService (ErrorLoggingMultiService):
# The ConnectionService is a MultiService which bundles all the connection # The ConnectionService is a MultiService which bundles all the connection
# services together for the purposes of being able to stop them and wait # services together for the purposes of being able to stop them and wait
# for all of their connections to close before shutting down. # for all of their connections to close before shutting down.
connectionServiceName = "ConnectionService" connectionServiceName = "cs"


def __init__(self, logObserver): def __init__(self, logObserver):
self.logObserver = logObserver # accesslog observer self.logObserver = logObserver # accesslog observer
Expand All @@ -292,6 +293,7 @@ def __init__(self, logObserver):
config.ErrorLogMaxRotatedFiles, config.ErrorLogMaxRotatedFiles,
config.ErrorLogRotateOnStart, config.ErrorLogRotateOnStart,
) )
self.name = "cds"




def privilegedStartService(self): def privilegedStartService(self):
Expand Down Expand Up @@ -862,6 +864,11 @@ def location(frame):
frame=frame frame=frame
) )


if config.Manhole.Enabled:
namespace= dict({service.name: service})
for n, s in service.namedServices.iteritems():
namespace[n] = s
self._makeManhole(namespace=namespace, parent=service)
return service return service




Expand Down Expand Up @@ -898,6 +905,7 @@ def makeService_Slave(self, options):
result = self.requestProcessingService(options, store, logObserver) result = self.requestProcessingService(options, store, logObserver)


if pool is not None: if pool is not None:
pool.setName("db")
pool.setServiceParent(result) pool.setServiceParent(result)


if config.ControlSocket: if config.ControlSocket:
Expand Down Expand Up @@ -937,6 +945,7 @@ def queueMasterAvailable(connectionFromMaster):
controlClient = ControlSocketConnectingService( controlClient = ControlSocketConnectingService(
endpointFactory, controlSocketClient endpointFactory, controlSocketClient
) )
controlClient.setName("control")
controlClient.setServiceParent(result) controlClient.setServiceParent(result)


# Optionally set up push notifications # Optionally set up push notifications
Expand All @@ -947,6 +956,7 @@ def queueMasterAvailable(connectionFromMaster):
pushSubService = ApplePushNotifierService.makeService( pushSubService = ApplePushNotifierService.makeService(
config.Notifications.Services.APNS, store) config.Notifications.Services.APNS, store)
observers.append(pushSubService) observers.append(pushSubService)
pushSubService.setName("APNS")
pushSubService.setServiceParent(result) pushSubService.setServiceParent(result)
if config.Notifications.Services.AMP.Enabled: if config.Notifications.Services.AMP.Enabled:
pushSubService = AMPPushForwarder(controlSocketClient) pushSubService = AMPPushForwarder(controlSocketClient)
Expand All @@ -959,6 +969,7 @@ def queueMasterAvailable(connectionFromMaster):
mailRetriever = MailRetriever( mailRetriever = MailRetriever(
store, directory, config.Scheduling.iMIP.Receiving store, directory, config.Scheduling.iMIP.Receiving
) )
mailRetriever.setName("MailRetriever")
mailRetriever.setServiceParent(result) mailRetriever.setServiceParent(result)
else: else:
mailRetriever = None mailRetriever = None
Expand Down Expand Up @@ -988,37 +999,6 @@ def decorateTransaction(txn):
txn._groupCacher = groupCacher txn._groupCacher = groupCacher


store.callWithNewTransactions(decorateTransaction) store.callWithNewTransactions(decorateTransaction)

# Optionally enable Manhole access
if config.Manhole.Enabled:
try:
from twisted.conch.manhole_tap import (
makeService as manholeMakeService
)
portString = "tcp:{:d}:interface=127.0.0.1".format(
config.Manhole.StartingPortNumber + int(config.LogID) + 1
)
manholeService = manholeMakeService({
"sshPort": None,
"telnetPort": portString,
"namespace": {
"config": config,
"service": result,
"store": store,
"directory": directory,
},
"passwd": config.Manhole.PasswordFilePath,
})
manholeService.setServiceParent(result)
# Using print(because logging isn't ready at this point)
print("Manhole access enabled:", portString)

except ImportError:
print(
"Manhole access could not enabled because "
"manhole_tap could not be imported"
)

return result return result




Expand Down Expand Up @@ -1181,10 +1161,12 @@ def updateFactory(configDict, reloading=False):
# 'SSL' tag on it, since that's the only time it's used. # 'SSL' tag on it, since that's the only time it's used.
contextFactory = None contextFactory = None


ReportingHTTPService( reportingService = ReportingHTTPService(
requestFactory, int(config.MetaFD), contextFactory, requestFactory, int(config.MetaFD), contextFactory,
usingSocketFile=config.SocketFiles.Enabled usingSocketFile=config.SocketFiles.Enabled,
).setServiceParent(connectionService) )
reportingService.setName("http-{}".format(int(config.MetaFD)))
reportingService.setServiceParent(connectionService)


else: # Not inheriting, therefore we open our own: else: # Not inheriting, therefore we open our own:
for bindAddress in self._allBindAddresses(): for bindAddress in self._allBindAddresses():
Expand All @@ -1211,6 +1193,8 @@ def updateFactory(configDict, reloading=False):
backlog=config.ListenBacklog, backlog=config.ListenBacklog,
inherit=False inherit=False
) )
httpsService.setName(
"https-{}:{}".format(bindAddress,int(port)))
httpsService.setServiceParent(connectionService) httpsService.setServiceParent(connectionService)


for port in config.BindHTTPPorts: for port in config.BindHTTPPorts:
Expand Down Expand Up @@ -1311,6 +1295,59 @@ def _spawnMemcached(self, monitor=None):
Popen(memcachedArgv) Popen(memcachedArgv)




def _makeManhole(self, namespace=None, parent=None):
try:
import inspect
import objgraph
except ImportError:
pass
try:
if 'inspect' in locals():
namespace['ins'] = inspect
if 'objgraph' in locals():
namespace['og'] = objgraph
from pprint import pprint
namespace.update({
'pp': pprint,
'cfg': config,
})
from twisted.conch.manhole_tap import (
makeService as manholeMakeService
)
portOffset = 0 if config.LogID == '' else int(config.LogID) + 1
portString = "tcp:{:d}:interface=127.0.0.1".format(
config.Manhole.StartingPortNumber + portOffset
)
manholeService = manholeMakeService({
"passwd": config.Manhole.PasswordFilePath,
"telnetPort":
portString if config.Manhole.UseSSH is False else None,
"sshPort":
portString if config.Manhole.UseSSH is True else None,
"sshKeyDir": config.DataRoot,
"sshKeyName": config.Manhole.sshKeyName,
"sshKeySize": config.Manhole.sshKeySize,
"namespace": namespace,
})
manholeService.setName("manhole")
if parent is not None:
manholeService.setServiceParent(parent)
# Using print(because logging isn't ready at this point)
print("Manhole access enabled:", portString)
except ImportError:
print(
"Manhole access could not enabled because "
"manhole_tap could not be imported."
)
import platform
if platform.system() == "Darwin":
if config.Manhole.UseSSH:
print(
"Set Manhole.UseSSH to false or rebuild CS with the "
"USE_OPENSSL environment variable set."
)


def makeService_Single(self, options): def makeService_Single(self, options):
""" """
Create a service to be used in a single-process, stand-alone Create a service to be used in a single-process, stand-alone
Expand All @@ -1332,6 +1369,7 @@ def slaveSvcCreator(pool, store, logObserver, storageService):
config.Notifications.Services.APNS, store config.Notifications.Services.APNS, store
) )
observers.append(pushSubService) observers.append(pushSubService)
pushSubService.setName("APNS")
pushSubService.setServiceParent(result) pushSubService.setServiceParent(result)
if config.Notifications.Services.AMP.Enabled: if config.Notifications.Services.AMP.Enabled:
pushSubService = AMPPushMaster( pushSubService = AMPPushMaster(
Expand Down Expand Up @@ -1362,6 +1400,7 @@ def slaveSvcCreator(pool, store, logObserver, storageService):
mailRetriever = MailRetriever( mailRetriever = MailRetriever(
store, directory, config.Scheduling.iMIP.Receiving store, directory, config.Scheduling.iMIP.Receiving
) )
mailRetriever.setName("mailRetriever")
mailRetriever.setServiceParent(result) mailRetriever.setServiceParent(result)
else: else:
mailRetriever = None mailRetriever = None
Expand Down Expand Up @@ -1401,35 +1440,6 @@ def slaveSvcCreator(pool, store, logObserver, storageService):
else: else:
groupCacher = None groupCacher = None


# Optionally enable Manhole access
if config.Manhole.Enabled:
try:
from twisted.conch.manhole_tap import (
makeService as manholeMakeService
)
portString = "tcp:{:d}:interface=127.0.0.1".format(
config.Manhole.StartingPortNumber
)
manholeService = manholeMakeService({
"sshPort": None,
"telnetPort": portString,
"namespace": {
"config": config,
"service": result,
"store": store,
"directory": directory,
},
"passwd": config.Manhole.PasswordFilePath,
})
manholeService.setServiceParent(result)
# Using print(because logging isn't ready at this point)
print("Manhole access enabled:", portString)
except ImportError:
print(
"Manhole access could not enabled because "
"manhole_tap could not be imported"
)

def decorateTransaction(txn): def decorateTransaction(txn):
txn._pushDistributor = pushDistributor txn._pushDistributor = pushDistributor
txn._rootResource = result.rootResource txn._rootResource = result.rootResource
Expand Down Expand Up @@ -1526,6 +1536,7 @@ def agentServiceCreator(pool, store, ignored, storageService):
config.ErrorLogMaxRotatedFiles, config.ErrorLogMaxRotatedFiles,
config.ErrorLogRotateOnStart, config.ErrorLogRotateOnStart,
) )
svc.setName("agent")
svc.setServiceParent(agentLoggingService) svc.setServiceParent(agentLoggingService)
return agentLoggingService return agentLoggingService


Expand Down Expand Up @@ -1582,6 +1593,7 @@ def subServiceFactory(connectionFactory, storageService):
dbtype=DatabaseType(dialect, paramstyle, dbfeatures), dbtype=DatabaseType(dialect, paramstyle, dbfeatures),
maxConnections=config.MaxDBConnectionsPerPool maxConnections=config.MaxDBConnectionsPerPool
) )
cp.setName("db")
cp.setServiceParent(ms) cp.setServiceParent(ms)
store = storeFromConfigWithoutDPS(config, cp.connection) store = storeFromConfigWithoutDPS(config, cp.connection)


Expand Down Expand Up @@ -1655,6 +1667,7 @@ def subServiceFactory(connectionFactory, storageService):
UpgradeReleaseLockStep(store) UpgradeReleaseLockStep(store)
) )


pps.setName("pre")
pps.setServiceParent(ms) pps.setServiceParent(ms)
return ms return ms


Expand Down Expand Up @@ -1784,13 +1797,15 @@ def makeService_Combined(self, options):


monitor = DelayedStartupProcessMonitor() monitor = DelayedStartupProcessMonitor()
s.processMonitor = monitor s.processMonitor = monitor
monitor.setName("pm")
monitor.setServiceParent(s) monitor.setServiceParent(s)


if config.MemoryLimiter.Enabled: if config.MemoryLimiter.Enabled:
memoryLimiter = MemoryLimitService( memoryLimiter = MemoryLimitService(
monitor, config.MemoryLimiter.Seconds, monitor, config.MemoryLimiter.Seconds,
config.MemoryLimiter.Bytes, config.MemoryLimiter.ResidentOnly config.MemoryLimiter.Bytes, config.MemoryLimiter.ResidentOnly
) )
memoryLimiter.setName("ml")
memoryLimiter.setServiceParent(s) memoryLimiter.setServiceParent(s)


# Maybe spawn memcached through a ProcessMonitor # Maybe spawn memcached through a ProcessMonitor
Expand Down Expand Up @@ -1877,32 +1892,6 @@ def _openSocket(addr, port):
statsService.setName("tcp-stats") statsService.setName("tcp-stats")
statsService.setServiceParent(s) statsService.setServiceParent(s)


# Optionally enable Manhole access
if config.Manhole.Enabled:
try:
from twisted.conch.manhole_tap import (
makeService as manholeMakeService
)
portString = "tcp:{:d}:interface=127.0.0.1".format(
config.Manhole.StartingPortNumber
)
manholeService = manholeMakeService({
"sshPort": None,
"telnetPort": portString,
"namespace": {
"config": config,
"service": s,
},
"passwd": config.Manhole.PasswordFilePath,
})
manholeService.setServiceParent(s)
# Using print(because logging isn't ready at this point)
print("Manhole access enabled:", portString)
except ImportError:
print(
"Manhole access could not enabled because "
"manhole_tap could not be imported"
)




# Finally, let's get the real show on the road. Create a service that # Finally, let's get the real show on the road. Create a service that
Expand Down Expand Up @@ -1944,11 +1933,13 @@ def spawnerSvcCreator(pool, store, ignored, storageService):
else: else:
dispenser = None dispenser = None
multi = MultiService() multi = MultiService()
multi.setName("multi")
pool.setServiceParent(multi) pool.setServiceParent(multi)
spawner = SlaveSpawnerService( spawner = SlaveSpawnerService(
self, monitor, dispenser, dispatcher, stats, options["config"], self, monitor, dispenser, dispatcher, stats, options["config"],
inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs inheritFDs=inheritFDs, inheritSSLFDs=inheritSSLFDs
) )
spawner.setName("spawner")
spawner.setServiceParent(multi) spawner.setServiceParent(multi)
if config.UseMetaFD: if config.UseMetaFD:
cl.setServiceParent(multi) cl.setServiceParent(multi)
Expand All @@ -1961,6 +1952,7 @@ def spawnerSvcCreator(pool, store, ignored, storageService):
mailRetriever = MailRetriever( mailRetriever = MailRetriever(
store, directory, config.Scheduling.iMIP.Receiving store, directory, config.Scheduling.iMIP.Receiving
) )
mailRetriever.setName("MailRetriever")
mailRetriever.setServiceParent(multi) mailRetriever.setServiceParent(multi)
else: else:
mailRetriever = None mailRetriever = None
Expand Down Expand Up @@ -1993,6 +1985,7 @@ def decorateTransaction(txn):
ssvc = self.storageService( ssvc = self.storageService(
spawnerSvcCreator, None, uid, gid spawnerSvcCreator, None, uid, gid
) )
ssvc.setName("ssvc")
ssvc.setServiceParent(s) ssvc.setServiceParent(s)
return s return s


Expand Down

0 comments on commit c4498a2

Please sign in to comment.