Skip to content
Browse files

fixes bug 855089 - remove redundant cron scripts, r=rhelmer,lonnen

  • Loading branch information...
1 parent ad4efc8 commit ce5026c3b302b5f0e9bff1fc870d369208d1fed0 @peterbe peterbe committed
View
54 scripts/config/bugzillaconfig.py.dist
@@ -1,54 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-bugzillaQuery = cm.Option()
-bugzillaQuery.doc = 'the URL to fetch a csv file from a query'
-bugzillaQuery.default = 'https://bugzilla.mozilla.org/buglist.cgi?query_format=advanced&short_desc_type=allwordssubstr&short_desc=&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&status_whiteboard_type=allwordssubstr&status_whiteboard=&keywords_type=allwords&keywords=&deadlinefrom=&deadlineto=&emailassigned_to1=1&emailtype1=substring&email1=&emailassigned_to2=1&emailreporter2=1&emailqa_contact2=1&emailcc2=1&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&chfieldfrom=%s&chfieldto=Now&chfield=[Bug+creation]&chfield=resolution&chfield=bug_status&chfield=short_desc&chfield=cf_crash_signature&chfieldvalue=&cmdtype=doit&order=Importance&field0-0-0=noop&type0-0-0=noop&value0-0-0=&columnlist=bug_id,bug_status,resolution,short_desc,cf_crash_signature&ctype=csv'
-
-persistentDataPathname = cm.Option()
-persistentDataPathname.doc = 'a pathname to a file system location where this script can store persistent data'
-persistentDataPathname.default = './bugzilla.pickle'
-
-daysIntoPast = cm.Option()
-daysIntoPast.doc = 'number of days to look into the past for bugs (0 - use last run time)'
-daysIntoPast.default = 0
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Bugzilla (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
68 scripts/config/buildsconfig.py.dist
@@ -1,68 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-import datetime
-
-from config.commonconfig import databaseHost
-from config.commonconfig import databasePort
-from config.commonconfig import databaseName
-from config.commonconfig import databaseUserName
-from config.commonconfig import databasePassword
-
-base_url = cm.Option()
-base_url.doc = 'The base url to use for fetching builds'
-base_url.default = 'http://stage.mozilla.org/pub/mozilla.org/'
-
-product_uris = cm.Option()
-product_uris.doc = 'a comma-delimited list of uris for each product'
-product_uris.default = 'firefox/nightly/latest-mozilla-1.9.2/,firefox/nightly/latest-mozilla-central/,seamonkey/nightly/latest-comm-1.9.1/,seamonkey/nightly/latest-comm-central-trunk/,thunderbird/nightly/latest-comm-1.9.2/,thunderbird/nightly/latest-comm-central/,mobile/nightly/latest-mobile-trunk/,camino/nightly/latest-2.1-M1.9.2/'
-product_uris.fromStringConverter = lambda x: tuple(x.split(','))
-
-platforms = cm.Option()
-platforms.doc = 'a comma-delimited list of platforms'
-platforms.default = 'linux-i686,linux-x86_64,mac,mac64,win32'
-platforms.fromStringConverter = lambda x: tuple(x.split(','))
-
-release_product_uris = cm.Option()
-release_product_uris.doc = 'a comma-delimited list of uris for each product'
-release_product_uris.default = 'firefox/nightly/,seamonkey/nightly/,thunderbird/nightly/,mobile/candidates/'
-release_product_uris.fromStringConverter = lambda x: tuple(x.split(','))
-
-release_platforms = cm.Option()
-release_platforms.doc = 'a comma-delimited list of platforms'
-release_platforms.default = 'linux,linux64,macosx64,macosx,win32,android,android-r7,linux-i686,macosx-i686,maemo5-gtk,win32-i686,linux-android,linux-maemo5-gtk,linux-mobile,macosx-mobile,win32-mobile'
-
-release_platforms.fromStringConverter = lambda x: tuple(x.split(','))
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Builds (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
View
42 scripts/config/dailyMatviewsConfig.py.dist
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Daily Matviews(pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
42 scripts/config/duplicatesconfig.py.dist
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Duplicate Detector (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
60 scripts/config/ftpscraperconfig.py.dist
@@ -1,60 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-import datetime
-
-from config.commonconfig import databaseHost
-from config.commonconfig import databasePort
-from config.commonconfig import databaseName
-from config.commonconfig import databaseUserName
-from config.commonconfig import databasePassword
-
-base_url = cm.Option()
-base_url.doc = 'The base url to use for fetching builds'
-base_url.default = 'http://ftp.mozilla.org/pub/mozilla.org'
-
-products = cm.Option()
-products.doc = 'a comma-delimited list of uris for each product'
-products.default = 'firefox,mobile,thunderbird,seamonkey,b2g'
-products.fromStringConverter = lambda x: tuple(x.split(','))
-
-backfillDate = cm.Option()
-backfillDate.doc = 'Date to backfill from, for example 2011-09-01'
-backfillDate.default = None
-
-#
-# Logging
-#
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Builds (pid %(process)d): \
- %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file \
- (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr \
- (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
View
42 scripts/config/hangReportconfig.py.dist
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Duplicate Detector (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
46 scripts/config/hbaseresubmitconfig.py.dist
@@ -1,46 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-from config.commonconfig import jsonFileSuffix
-from config.commonconfig import dumpFileSuffix
-
-from config.collectorconfig import hbaseFallbackDumpPermissions
-from config.collectorconfig import hbaseFallbackDirPermissions
-from config.collectorconfig import hbaseFallbackDumpGID
-from config.collectorconfig import hbaseFallbackDumpDirCount
-from config.collectorconfig import hbaseFallbackFS
-from config.commonconfig import hbaseHost
-from config.commonconfig import hbasePort
-from config.commonconfig import hbaseTimeout
-
-logFilePathname = cm.Option()
-logFilePathname.doc = 'full pathname for the log file'
-logFilePathname.default = './hbaseresubmit.log'
-
-logFileMaximumSize = cm.Option()
-logFileMaximumSize.doc = 'maximum size in bytes of the log file'
-logFileMaximumSize.default = 1000000
-
-logFileMaximumBackupHistory = cm.Option()
-logFileMaximumBackupHistory.doc = 'maximum number of log files to keep'
-logFileMaximumBackupHistory.default = 50
-
-logFileLineFormatString = cm.Option()
-logFileLineFormatString.doc = 'python logging system format for log file entries'
-logFileLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-logFileErrorLoggingLevel = cm.Option()
-logFileErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-logFileErrorLoggingLevel.default = 20
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 40
-
View
42 scripts/config/reportsclean.py.dist
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Reports Clean (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
46 scripts/config/signaturesconfig.py.dist
@@ -1,46 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-from config.commonconfig import databaseHost
-from config.commonconfig import databasePort
-from config.commonconfig import databaseName
-from config.commonconfig import databaseUserName
-from config.commonconfig import databasePassword
-
-start_day= cm.Option()
-start_day.doc = 'The number of days before today that should serve as the start date for the query window.'
-start_day.default = 1
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Signatures (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 40
View
65 scripts/config/topCrashesBySignatureConfig.py.dist
@@ -1,65 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-from config.commonconfig import debug
-
-from config.commonconfig import databaseHost
-from config.commonconfig import databaseName
-from config.commonconfig import databaseUserName
-from config.commonconfig import databasePassword
-
-from config.commonconfig import startWindow
-from config.commonconfig import endWindow
-from config.commonconfig import deltaWindow
-from config.commonconfig import defaultDeltaWindow
-defaultDeltaWindow.default = '00:12:00'
-
-from config.commonconfig import startDate
-from config.commonconfig import endDate
-from config.commonconfig import deltaDate
-from config.commonconfig import initialDeltaDate
-initialDeltaDate.default = '96:00:00'
-
-product = cm.Option()
-product.doc = 'the name of the product to restrict to (leave blank for all)'
-product.default = 'Firefox'
-
-version = cm.Option()
-version.doc = 'the name of the version to restrict to (leave blank for all)'
-version.default = ''
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Top Crashes By Signature (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
-
View
73 scripts/config/topCrashesByUrlConfig.py.dist
@@ -1,73 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-import datetime
-
-from config.commonconfig import databaseHost
-from config.commonconfig import databaseName
-from config.commonconfig import databaseUserName
-from config.commonconfig import databasePassword
-
-from config.commonconfig import startWindow
-from config.commonconfig import endWindow
-from config.commonconfig import deltaWindow
-from config.commonconfig import defaultDeltaWindow
-defaultDeltaWindow.default = '24:00:00'
-
-from config.commonconfig import startDate
-from config.commonconfig import endDate
-from config.commonconfig import deltaDate
-from config.commonconfig import initialDeltaDate
-initialDeltaDate.default = '96:00:00'
-
-truncateUrlLength = cm.Option()
-truncateUrlLength.doc = 'The maximum stored length of a URL (leave None for no truncation)'
-truncateUrlLength.default = None
-
-product = cm.Option()
-product.doc = 'the name of the product to restrict to (leave blank for all)'
-product.default = 'Firefox'
-
-version = cm.Option()
-version.doc = 'the name of the version to restrict to (leave blank for all)'
-version.default = ''
-
-from config.commonconfig import debug
-
-processingDay = cm.Option()
-processingDay.doc = 'One day to process in (YYYY-MM-DD) format'
-processingDay.singleCharacter = 'd'
-processingDay.fromStringConverter = cm.dateTimeConverter
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Top Crashes by URL (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 40
View
42 scripts/config/updateadus.py.dist
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import socorro.lib.ConfigurationManager as cm
-
-try:
- from config.commonconfig import *
-except ImportError:
- from commonconfig import *
-
-#-------------------------------------------------------------------------------
-# Logging
-
-syslogHost = cm.Option()
-syslogHost.doc = 'syslog hostname'
-syslogHost.default = 'localhost'
-
-syslogPort = cm.Option()
-syslogPort.doc = 'syslog port'
-syslogPort.default = 514
-
-syslogFacilityString = cm.Option()
-syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
-syslogFacilityString.default = 'user'
-
-syslogLineFormatString = cm.Option()
-syslogLineFormatString.doc = 'python logging system format for syslog entries'
-syslogLineFormatString.default = 'Socorro Update ADUs (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
-
-syslogErrorLoggingLevel = cm.Option()
-syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-syslogErrorLoggingLevel.default = 10
-
-stderrLineFormatString = cm.Option()
-stderrLineFormatString.doc = 'python logging system format for logging to stderr'
-stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
-
-stderrErrorLoggingLevel = cm.Option()
-stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
-stderrErrorLoggingLevel.default = 10
-
View
15 scripts/crons/cron_aggregates.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startTopCrashesBySignature
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_bugzilla.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startBugzilla
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
45 scripts/crons/cron_clean_tcbs.sh
@@ -1,45 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# Periodically clean top_crash_by_signature table
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-
-if [[ ! "$databaseHost" && "$databaseUserName" && "$databaseName" ]]
-then
- echo "error: need databaseHost, databaseUserName, databaseName set in $SOCORRO_CONFIG"
- exit 1
-fi
-
-lock $NAME
-
-pyjob $NAME createPartitions
-EXIT_CODE=$?
-if [ $EXIT_CODE != 0 ]
-then
- echo "createPartitions failed, exiting $NAME early!"
- unlock $NAME
- exit $EXIT_CODE
-fi
-
-psql -t -h ${databaseHost} -U ${databaseUserName} ${databaseName} <<SQL_END
-delete from top_crashes_by_signature
-where id in
- (select
- tcbs.id
- from
- top_crashes_by_signature tcbs
- join product_visibility pv on tcbs.productdims_id = pv.productdims_id
- where
- tcbs.window_end < (case when now() < pv.end_date then now()
- else pv.end_date end) - interval '56 days');
-SQL_END
-EXIT_CODE=$?
-
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_daily_crashes.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startDailyCrash
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_daily_matviews.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startDailyMatviews
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_duplicates.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startDuplicates
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_ftpscraper.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startFtpScraper
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_reportsclean.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startReportsClean
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_tcurl.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startTopCrashesByUrl
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
15 scripts/crons/cron_update_adus.sh
@@ -1,15 +0,0 @@
-#!/bin/bash
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-. /etc/socorro/socorrorc
-
-NAME=`basename $0 .sh`
-lock $NAME
-pyjob $NAME startUpdateADUs
-EXIT_CODE=$?
-unlock $NAME
-
-exit $EXIT_CODE
View
145 socorro/cron/bugzilla.py
@@ -1,145 +0,0 @@
-#!/usr/bin/python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import urllib2
-import logging
-import datetime as dt
-import cPickle
-import csv
-
-logger = logging.getLogger("bugzilla")
-
-import socorro.lib.psycopghelper as psy
-import socorro.lib.util as util
-
-from socorro.lib.datetimeutil import utc_now
-
-#-----------------------------------------------------------------------------------------------------------------
-
-def signature_set_from_string(signatureString):
- signatureSet = set()
- try:
- sigStart = 0
- sigEnd = 0
- while True:
- sigStart = signatureString.index("[@", sigEnd) + 2
- sigEnd = signatureString.index("]", sigEnd + 1)
- signatureSet.add(signatureString[sigStart:sigEnd].strip())
- except ValueError:
- # throw when index cannot match another sig, ignore
- pass
- return signatureSet
-
-#-----------------------------------------------------------------------------------------------------------------
-
-def bugzilla_iterator(query, querySourceFunction=urllib2.urlopen):
- logger.debug("query: %s", query)
- for bugReport in csv.DictReader(querySourceFunction(query)):
- logger.debug("reading csv: %s", str(bugReport))
- yield (int(bugReport["bug_id"]),
- bugReport["bug_status"],
- bugReport["resolution"],
- bugReport["short_desc"],
- signature_set_from_string(bugReport["cf_crash_signature"]))
-
-#-----------------------------------------------------------------------------------------------------------------
-def signature_is_found(signature, databaseCursor):
- try:
- psy.singleValueSql(databaseCursor, "select id from reports where signature = %s limit 1", (signature,))
- return True
- except psy.SQLDidNotReturnSingleValue:
- return False
-
-#-----------------------------------------------------------------------------------------------------------------
-def insert_or_update_bug_in_database(bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureSetFromBugzilla,
- databaseCursor, signatureFoundInReportsFunction=signature_is_found):
- try:
- if len(signatureSetFromBugzilla) == 0:
- databaseCursor.execute("delete from bugs where id = %s", (bugId,))
- databaseCursor.connection.commit()
- logger.info("rejecting bug (no signatures): %s - %s, %s", bugId, statusFromBugzilla, resolutionFromBugzilla)
- else:
- useful = False
- insertMade = False
- try:
- statusFromDatabase, resolutionFromDatabase, shortDescFromDatabase = psy.singleRowSql(databaseCursor, "select status, resolution, short_desc from bugs where id = %s", (bugId,))
- if statusFromDatabase != statusFromBugzilla or resolutionFromDatabase != resolutionFromBugzilla or shortDescFromDatabase != shortDescFromBugzilla:
- databaseCursor.execute("update bugs set status = %s, resolution = %s, short_desc = %s where id = %s", (statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, bugId))
- logger.info("bug status updated: %s - %s, %s", bugId, statusFromBugzilla, resolutionFromBugzilla)
- useful = True
- listOfSignaturesFromDatabase = [x[0] for x in psy.execute(databaseCursor, "select signature from bug_associations where bug_id = %s", (bugId,))]
- for aSignature in listOfSignaturesFromDatabase:
- if aSignature not in signatureSetFromBugzilla:
- databaseCursor.execute("delete from bug_associations where signature = %s and bug_id = %s", (aSignature, bugId))
- logger.info ('association removed: %s - "%s"', bugId, aSignature)
- useful = True
- except psy.SQLDidNotReturnSingleRow:
- databaseCursor.execute("insert into bugs (id, status, resolution, short_desc) values (%s, %s, %s, %s)", (bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla))
- insertMade = True
- listOfSignaturesFromDatabase = []
- for aSignature in signatureSetFromBugzilla:
- if aSignature not in listOfSignaturesFromDatabase:
- if signatureFoundInReportsFunction(aSignature, databaseCursor):
- databaseCursor.execute("insert into bug_associations (signature, bug_id) values (%s, %s)", (aSignature, bugId))
- logger.info ('new association: %s - "%s"', bugId, aSignature)
- useful = True
- else:
- logger.info ('rejecting association (no reports with this signature): %s - "%s"', bugId, aSignature)
- if useful:
- databaseCursor.connection.commit()
- if insertMade:
- logger.info('new bug: %s - %s, %s, "%s"', bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla)
- else:
- databaseCursor.connection.rollback()
- if insertMade:
- logger.info('rejecting bug (no useful information): %s - %s, %s, "%s"', bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla)
- else:
- logger.info('skipping bug (no new information): %s - %s, %s, "%s"', bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla)
- except Exception, x:
- databaseCursor.connection.rollback()
- raise
-
-#-----------------------------------------------------------------------------------------------------------------
-def get_last_run_date(config, now_function=utc_now):
- if config.daysIntoPast == 0:
- try:
- f = open(config.persistentDataPathname)
- try:
- return cPickle.load(f)
- finally:
- f.close()
- except IOError:
- return now_function() - dt.timedelta(days=30)
- else:
- return now_function() - dt.timedelta(days=config.daysIntoPast)
-
-#-----------------------------------------------------------------------------------------------------------------
-def save_last_run_date(config, now_function=utc_now):
- try:
- f = open(config.persistentDataPathname, "w")
- try:
- return cPickle.dump(now_function(), f)
- finally:
- f.close()
- except IOError:
- util.reportExceptionAndContinue(logger)
-
-#-----------------------------------------------------------------------------------------------------------------
-def record_associations(config):
- databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger)
- try:
- databaseConnection, databaseCursor = databaseConnectionPool.connectionCursorPair()
- lastRunDate = get_last_run_date(config)
- lastRunDateAsString = lastRunDate.strftime('%Y-%m-%d')
- logger.info("beginning search from this date (YYYY-MM-DD): %s", lastRunDateAsString)
- query = config.bugzillaQuery % lastRunDateAsString
- for bug, status, resolution, short_desc, signatureSet in bugzilla_iterator(query):
- logger.debug("bug %s (%s, %s) %s: %s", bug, status, resolution, short_desc, signatureSet)
- insert_or_update_bug_in_database (bug, status, resolution, short_desc, signatureSet, databaseCursor)
- save_last_run_date(config)
- finally:
- databaseConnectionPool.cleanup()
-
View
90 socorro/cron/dailyMatviews.py
@@ -1,90 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import sys
-import logging
-import datetime
-import psycopg2
-
-from socorro.lib.datetimeutil import utc_now
-
-logger = logging.getLogger('dailyMatviews')
-logger.addHandler(logging.StreamHandler(sys.stderr))
-
-
-def update(config, targetDate):
- functions = (
- # function name, parmeters, dependencies
- ('update_product_versions', [], []),
- ('update_signatures', [targetDate], []),
- ('update_os_versions', [targetDate], []),
- ('update_tcbs', [targetDate], []),
- ('update_adu', [targetDate], []),
- ('update_rank_compare', [targetDate], []),
- ('update_nightly_builds', [targetDate], []),
- ('update_build_adu', [targetDate], []),
- ('update_crashes_by_user', [targetDate], ['update_adu']),
- ('update_crashes_by_user_build', [targetDate], []),
- ('update_correlations', [targetDate], []),
- ('update_home_page_graph', [targetDate], ['update_adu']),
- ('update_home_page_graph_build', [targetDate], ['update_build_adu']),
- ('update_tcbs_build', [targetDate], []),
- ('update_explosiveness', [targetDate],
- ['update_tcbs', 'update_build_adu']),
- )
-
- failed = set()
- # returning false is ok in some cases, see bug 790022
- failureOk = ('update_adu',)
- databaseDSN = ""
- if 'databaseHost' in config:
- databaseDSN += 'host=%(databaseHost)s '
- if 'databaseName' in config:
- databaseDSN += 'dbname=%(databaseName)s '
- if 'databaseUserName' in config:
- databaseDSN += 'user=%(databaseUserName)s '
- if 'databasePassword' in config:
- databaseDSN += 'password=%(databasePassword)s'
- dsn = databaseDSN % config
- connection = psycopg2.connect(dsn)
- cursor = connection.cursor()
- for funcname, parameters, deps in functions:
- if set(deps) & failed:
- # one of the deps previously failed, so skip this one
- logger.warn("For %r, dependency %s failed so skipping"
- % (funcname, ', '.join(set(deps) & failed)))
- continue
- logger.info('Running %s' % funcname)
- failureMessage = None
- success = False
- # Set the temp_buffers for this session
- databaseTempbuffers = '8MB' # default
- if 'databaseTempbuffers' in config:
- databaseTempbuffers = config.databaseTempbuffers
- cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers,));
- try:
- cursor.callproc(funcname, parameters)
- # fetchone() returns a tuple of length 1
- result = cursor.fetchone()
- if result and result[0] or funcname in failureOk:
- success = True
- else:
- # "expected" error
- logger.warn('%r failed' % funcname)
- failureMessage = '%s did not return true' % funcname
- except psycopg2.InternalError:
- # unexpected error
- logger.error('%r failed' % funcname, exc_info=True)
- import sys # don't assume that this has been imported
- __, error_value = sys.exc_info()[:2]
- failureMessage = str(error_value)
- if success:
- connection.commit()
- else:
- connection.rollback()
- failed.add(funcname)
-
- return len(failed)
View
40 socorro/cron/duplicates.py
@@ -1,40 +0,0 @@
-#!/usr/bin/python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import logging
-from datetime import datetime
-from datetime import timedelta
-
-logger = logging.getLogger("duplicates")
-
-import socorro.lib.psycopghelper as psy
-import socorro.lib.util as util
-
-from socorro.lib.datetimeutil import utc_now
-
-#-----------------------------------------------------------------------------------------------------------------
-def find_duplicates(config):
- databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger)
- # Set the temp_buffers for this session
- databaseTempbuffers = '8MB' # default
- if 'databaseTempbuffers' in config:
- databaseTempbuffers = config.databaseTempbuffers
- try:
- connection, cursor= databaseConnectionPool.connectionCursorPair()
-
- cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers,));
- startTime = utc_now() - timedelta(hours=3)
- endTime = startTime + timedelta(hours=1)
- cursor.callproc('update_reports_duplicates', (startTime, endTime))
- connection.commit()
-
- startTime += timedelta(minutes=30)
- endTime = startTime + timedelta(hours=1)
- cursor.callproc('update_reports_duplicates', (startTime, endTime))
- connection.commit()
- finally:
- databaseConnectionPool.cleanup()
-
View
289 socorro/cron/ftpscraper.py
@@ -1,289 +0,0 @@
-#! /usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-ftpscraper.py pulls build information from ftp.mozilla.org for
-nightly and release builds.
-"""
-import logging
-import urllib2
-import lxml.html
-import datetime
-import json
-
-import socorro.lib.buildutil as buildutil
-import socorro.lib.psycopghelper as psy
-import socorro.lib.util as util
-
-logger = logging.getLogger("ftpscraper")
-"""
- Socket timeout to prevent FTP from hanging indefinitely
- Picked a 2 minute timeout as a generous allowance,
- given the entire script takes about that much time to run.
-"""
-import socket
-socket.setdefaulttimeout(120)
-
-def getLinks(url, startswith=None, endswith=None, urllib=urllib2):
- page = urllib.urlopen(url)
- html = lxml.html.document_fromstring(page.read())
- page.close()
-
- results = []
- for element, attribute, link, pos in html.iterlinks():
- if startswith:
- if link.startswith(startswith):
- results.append(link)
- elif endswith:
- if link.endswith(endswith):
- results.append(link)
- return results
-
-
-def parseInfoFile(url, nightly=False, urllib=urllib2):
- infotxt = urllib.urlopen(url)
- contents = infotxt.read().split()
- infotxt.close()
-
- results = {}
- if nightly:
- results = {'buildID': contents[0], 'rev': contents[1]}
- if len(contents) > 2:
- results['altrev'] = contents[2]
- else:
- for entry in contents:
- (k, v) = entry.split('=')
- results[k] = v
-
- return results
-
-"""
- Parse the B2G manifest JSON file
- Example: {"buildid": "20130125070201", "update_channel": "nightly", "version": "18.0"}
- TODO handle exception if file does not exist
-"""
-def parseB2GFile(url, nightly=False, urllib=urllib2):
- infotxt = urllib.urlopen(url)
- results = json.load(infotxt)
- infotxt.close()
-
- # XXX KLUDGE Default 'null' channels to nightly
- if results['update_channel']:
- results['build_type'] = results['update_channel']
- else:
- results['build_type'] = 'nightly'
-
- # XXX KLUDGE Default beta_number to 1 for beta releases
- if results['update_channel'] == 'beta':
- results['beta_number'] = results.get('beta_number', 1)
-
- return results
-
-
-def getRelease(dirname, url, urllib=urllib2):
- candidate_url = '%s/%s' % (url, dirname)
- builds = getLinks(candidate_url, startswith='build', urllib=urllib)
- if not builds:
- logger.info('No build dirs in %s' % candidate_url)
- return
-
- latest_build = builds.pop()
- build_url = '%s/%s' % (candidate_url, latest_build)
-
- info_files = getLinks(build_url, endswith='_info.txt', urllib=urllib)
-
- for f in info_files:
- info_url = '%s/%s' % (build_url, f)
- kvpairs = parseInfoFile(info_url)
-
- platform = f.split('_info.txt')[0]
-
- version = dirname.split('-candidates')[0]
- build_number = latest_build.strip('/')
-
- yield (platform, version, build_number, kvpairs)
-
-
-def getNightly(dirname, url, urllib=urllib2, backfill_date=None):
- nightly_url = '%s/%s' % (url, dirname)
-
- info_files = getLinks(nightly_url, endswith='.txt', urllib=urllib)
- for f in info_files:
- pv = None
- platform = None
- if 'en-US' in f:
- (pv, platform) = f.strip('.txt').split('.en-US.')
- elif 'multi' in f:
- (pv, platform) = f.strip('.txt').split('.multi.')
- else:
- return
-
- version = pv.split('-')[-1]
- repository = []
-
- for field in dirname.split('-'):
- if not field.isdigit():
- repository.append(field)
- repository = '-'.join(repository).strip('/')
-
- info_url = '%s/%s' % (nightly_url, f)
- kvpairs = parseInfoFile(info_url, nightly=True)
-
- yield (platform, repository, version, kvpairs)
-
-"""
- Last mile of B2G scraping, calls parseB2G on .json
- Files look like: socorro_unagi-stable_2013-01-25-07.json
-"""
-def getB2G(dirname, url, urllib=urllib2, backfill_date=None):
- url = '%s/%s' % (url, dirname)
- info_files = getLinks(url, endswith='.json', urllib=urllib)
- platform = None
- version = None
- repository = 'b2g-release'
- for f in info_files:
- # Pull platform out of the filename
- jsonfilename = f.split('.json')[0].split('_')
- # Skip if this file isn't for socorro!
- if jsonfilename[0] != 'socorro':
- next
- platform = jsonfilename[1]
-
- info_url = '%s/%s' % (url, f)
- kvpairs = parseB2GFile(info_url, nightly=True)
- version = kvpairs['version']
- # Hardcoding per IRC convo 1/30/2013
- yield (platform, repository, version, kvpairs)
-
-
-def recordBuilds(config, backfill_date):
- databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost,
- config.databaseName, config.databaseUserName, config.databasePassword,
- logger)
-
- try:
- connection, cursor = databaseConnectionPool.connectionCursorPair()
- for product_name in config.products:
- if product_name == 'b2g':
- today = datetime.datetime.today()
- if backfill_date is not None:
- currentdate = backfill_date
- while currentdate <= today:
- logger.debug('backfilling for date ' + str(currentdate))
- scrapeB2G(config, cursor, product_name,
- date=currentdate)
- currentdate += datetime.timedelta(days=1)
- else:
- scrapeB2G(config, cursor, product_name, date=today)
- else:
-
- scrapeReleases(config, cursor, product_name)
-
- today = datetime.datetime.today()
- if backfill_date is not None:
- currentdate = backfill_date
- while currentdate <= today:
- logger.debug('backfilling for date ' + str(currentdate))
- scrapeNightlies(config, cursor, product_name,
- date=currentdate)
- currentdate += datetime.timedelta(days=1)
- else:
- scrapeNightlies(config, cursor, product_name, date=today)
- finally:
- databaseConnectionPool.cleanup()
-
-
-def scrapeReleases(config, cursor, product_name, urllib=urllib2):
- prod_url = '%s/%s/' % (config.base_url, product_name)
-
- # releases are sometimes in nightly, sometimes in candidates dir.
- # look in both.
- for directory in ('nightly', 'candidates'):
- if not getLinks(prod_url, startswith=directory, urllib=urllib):
- logger.debug('Dir %s not found for %s' % (directory, product_name))
- continue
-
- url = '%s/%s/%s/' % (config.base_url, product_name, directory)
-
- try:
- releases = getLinks(url, endswith='-candidates/',
- urllib=urllib)
- for release in releases:
- for info in getRelease(release, url):
- (platform, version, build_number, kvpairs) = info
- build_type = 'Release'
- beta_number = None
- repository = 'mozilla-release'
- if 'b' in version:
- build_type = 'Beta'
- version, beta_number = version.split('b')
- repository = 'mozilla-beta'
- build_id = kvpairs['buildID']
- buildutil.insert_build(cursor, product_name, version,
- platform, build_id, build_type,
- beta_number, repository,
- ignore_duplicates=True)
- except urllib.URLError:
- util.reportExceptionAndContinue(logger)
-
-
-"""
- Connect to and scrape info for B2G releases
- URL for b2g: stage.mozilla.org/pub/b2g/manifests/2013/01/2013-01-25-07/
- Ignoring 'latest' for now
- In the style of scrapeNightlies
-"""
-def scrapeB2G(config, cursor, product_name, urllib=urllib2, date=None):
- month = date.strftime('%m')
- b2g_url = '%s/%s/%s/' % (config.base_url, product_name,
- 'manifests')
-
- try:
- day = date.strftime('%d')
- dir_prefix = '%s-%s-%s' % (date.year, month, day)
- # I have no idea what this first level of directories means :/
- # TODO get info about that and update this search
- version_dirs = getLinks(b2g_url, startswith='1.', urllib=urllib)
- for version_dir in version_dirs:
- # /1.0.0/2013/01/2013-01-27-07/*.json
- prod_url = '%s/%s/%s/%s/' % (b2g_url, version_dir, date.year, month)
- nightlies = getLinks(prod_url, startswith=dir_prefix, urllib=urllib)
- for nightly in nightlies:
- for info in getB2G(nightly, prod_url):
- (platform, repository, version, kvpairs) = info
- build_id = kvpairs['buildid']
- build_type = kvpairs['build_type']
- buildutil.insert_build(cursor, product_name, version, platform,
- build_id, build_type, kvpairs.get('beta_number', None), repository,
- ignore_duplicates=True)
-
- except urllib.URLError:
- util.reportExceptionAndContinue(logger)
-
-
-def scrapeNightlies(config, cursor, product_name, urllib=urllib2, date=None):
- month = date.strftime('%m')
- nightly_url = '%s/%s/%s/%s/%s/' % (config.base_url, product_name,
- 'nightly', date.year, month)
-
- try:
-
- day = date.strftime('%d')
- dir_prefix = '%s-%s-%s' % (date.year, month, day)
- nightlies = getLinks(nightly_url, startswith=dir_prefix,
- urllib=urllib)
- for nightly in nightlies:
- for info in getNightly(nightly, nightly_url):
- (platform, repository, version, kvpairs) = info
- build_id = kvpairs['buildID']
- build_type = 'Nightly'
- if version.endswith('a2'):
- build_type = 'Aurora'
- buildutil.insert_build(cursor, product_name, version, platform,
- build_id, build_type, None, repository,
- ignore_duplicates=True)
-
- except urllib.URLError:
- util.reportExceptionAndContinue(logger)
View
31 socorro/cron/reportsClean.py
@@ -1,31 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import logging
-from datetime import datetime
-from datetime import timedelta
-
-logger = logging.getLogger("reports_clean")
-
-import socorro.lib.psycopghelper as psy
-import socorro.lib.util as util
-
-from socorro.lib.datetimeutil import utc_now
-
-#-----------------------------------------------------------------------------------------------------------------
-def update_reports_clean(config):
- databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost, config.databaseName, config.databaseUserName, config.databasePassword, logger)
- # Set the temp_buffers for this session
- databaseTempbuffers = '8MB' # default
- if 'databaseTempbuffers' in config:
- databaseTempbuffers = config.databaseTempbuffers
- try:
- connection, cursor= databaseConnectionPool.connectionCursorPair()
- cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers,));
- startTime = utc_now() - timedelta(hours=2)
- cursor.callproc('update_reports_clean', [startTime])
- connection.commit()
- finally:
- databaseConnectionPool.cleanup()
-
View
39 socorro/cron/updateADUs.py
@@ -1,39 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import logging
-import datetime
-
-logger = logging.getLogger("updateADUs")
-
-import psycopg2
-import socorro.lib.psycopghelper as psy
-import socorro.lib.util as util
-
-from socorro.lib.datetimeutil import utc_now
-
-def update_adus(config):
- databaseConnectionPool = psy.DatabaseConnectionPool(config.databaseHost,
- config.databaseName, config.databaseUserName, config.databasePassword,
- logger)
- # Set the temp_buffers for this session
- databaseTempbuffers = '8MB' # default
- if 'databaseTempbuffers' in config:
- databaseTempbuffers = config.databaseTempbuffers
- try:
- connection, cursor = databaseConnectionPool.connectionCursorPair()
-
- cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers,));
- startTime = (utc_now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
- cursor.callproc('update_adu', [startTime])
- connection.commit()
- except psycopg2.InternalError, e:
- errmsg = 'ERROR: update_adu has already been run for %s\n' % (startTime)
- if e.pgerror == errmsg:
- logger.warn('Update ADU already run for %s, ignoring' % (startTime))
- else:
- raise
- finally:
- databaseConnectionPool.cleanup()
-
View
292 socorro/unittest/cron/testBugzilla.py
@@ -1,292 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import datetime as dt
-import errno
-import logging
-import os
-import time
-
-import psycopg2
-
-import unittest
-from nose.tools import *
-
-import socorro.lib.ConfigurationManager as configurationManager
-import socorro.lib.util
-import socorro.lib.psycopghelper as psy
-import socorro.cron.bugzilla as bug
-import socorro.database.schema as sch
-import socorro.database.database as sdatabase
-
-from socorro.lib.datetimeutil import UTC
-
-#from createTables import createCronTables, dropCronTables
-import socorro.unittest.testlib.dbtestutil as dbtestutil
-from socorro.unittest.testlib.loggerForTest import TestingLogger
-from socorro.unittest.testlib.testDB import TestDB
-import socorro.unittest.testlib.util as tutil
-
-import cronTestconfig as testConfig
-
-def makeBogusReports (connection, cursor, logger):
- # make some bogus data in the reports table
- reportsTable = sch.ReportsTable(logger)
- # ( uuid, client_crash_date, date_processed, product, version, build, url, install_age, last_crash, uptime, email, user_id, user_comments, app_notes, distributor, distributor_version, topmost_filenames, addons_checked, flash_version, hangid, process_type) values
- fakeReportData = [ (( "uuid1", None, dt.datetime(2009, 05, 04, tzinfo=UTC), "bogus", "1.0", "xxx", "http://cnn.com", 100, 14, 10, None, None, "bogus", "", "", ",", None, None, None, None, None, 'release'), "BogusClass::bogus_signature (const char**, void *)"),
- (( "uuid2", None, dt.datetime(2009, 05, 04, tzinfo=UTC), "bogus", "1.0", "xxx", "http://cnn.com", 100, 14, 10, None, None, "bogus", "", "", ",", None, None, None, None, None, 'release'), "js3250.dll@0x6cb96"),
- (( "uuid3", None, dt.datetime(2009, 05, 04, tzinfo=UTC), "bogus", "1.0", "xxx", "http://cnn.com", 100, 14, 10, None, None, "bogus", "", "", ",", None, None, None, None, None, 'release'), "libobjc.A.dylib@0x1568c"),
- (( "uuid4", None, dt.datetime(2009, 05, 04, tzinfo=UTC), "bogus", "1.0", "xxx", "http://cnn.com", 100, 14, 10, None, None, "bogus", "", "", ",", None, None, None, None, None, 'release'), "nanojit::LIns::isTramp()"),
- (( "uuid5", None, dt.datetime(2009, 05, 04, tzinfo=UTC), "bogus", "1.0", "xxx", "http://cnn.com", 100, 14, 10, None, None, "bogus", "", "", ",", None, None, None, None, None, 'release'), "libobjc.A.dylib@0x1568c"),
- ]
- try:
- #altconn = psycopg2.connect(me.dsn)
- altconn = me.database.connection()
- altcur = altconn.cursor()
- except Exception, x:
- print "Exception at line 40:",type(x),x
- raise
- def cursorFunction():
- return altconn, altcur
- for rep, sig in fakeReportData:
- try:
- reportsTable.insert(cursor, rep, cursorFunction, date_processed=rep[2])
- connection.commit()
- cursor.execute("update reports set signature=%s where date_processed = %s and uuid = %s", (sig, rep[2], rep[0]))
- connection.commit()
- except Exception, x:
- print "Exception at line 51", type(x),x
- connection.rollback()
- altconn.close()
-
-class Me: # not quite "self"
- """
- I need stuff to be initialized once per module. Rather than having a bazillion globals, lets just have 'me'
- """
- pass
-me = None
-
-def setup_module():
- global me
- if me:
- return
- me = Me()
- me.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')
- tutil.nosePrintModule(__file__)
- myDir = os.path.split(__file__)[0]
- if not myDir: myDir = '.'
- replDict = {'testDir':'%s'%myDir}
- for i in me.config:
- try:
- me.config[i] = me.config.get(i)%(replDict)
- except:
- pass
- me.logFilePathname = me.config.logFilePathname
- if not me.logFilePathname:
- me.logFilePathname = 'logs/bugzilla_test.log'
- logFileDir = os.path.split(me.logFilePathname)[0]
- try:
- os.makedirs(logFileDir)
- except OSError,x:
- if errno.EEXIST == x.errno: pass
- else: raise
- f = open(me.logFilePathname,'w')
- f.close()
- fileLog = logging.FileHandler(me.logFilePathname, 'a')
- fileLog.setLevel(logging.DEBUG)
- fileLogFormatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
- fileLog.setFormatter(fileLogFormatter)
- me.fileLogger = logging.getLogger("bugzilla")
- me.fileLogger.addHandler(fileLog)
- # Got trouble? See what's happening by uncommenting the next three lines
- #stderrLog = logging.StreamHandler()
- #stderrLog.setLevel(10)
- #me.fileLogger.addHandler(stderrLog)
-
- me.database = sdatabase.Database(me.config)
- #me.dsn = "host=%s dbname=%s user=%s password=%s" % (me.config.databaseHost,me.config.databaseName,
- #me.config.databaseUserName,me.config.databasePassword)
- me.testDB = TestDB()
- me.testDB.removeDB(me.config,me.fileLogger)
- me.testDB.createDB(me.config,me.fileLogger)
- try:
- me.conn = me.database.connection()
- #me.conn = psycopg2.connect(me.dsn)
- #me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
- me.cur = me.conn.cursor()
- except Exception, x:
- print "Exception at line 107",type(x),x
- socorro.lib.util.reportExceptionAndAbort(me.fileLogger)
- makeBogusReports(me.conn, me.cur, me.fileLogger)
-
-def teardown_module():
- global me
- me.testDB.removeDB(me.config,me.fileLogger)
- me.conn.close()
- try:
- os.unlink(me.logFilePathname)
- except:
- pass
-
-class TestBugzilla(unittest.TestCase):
- def setUp(self):
- global me
- self.config = configurationManager.newConfiguration(configurationModule = testConfig, applicationName='Testing bugzilla')
-
- myDir = os.path.split(__file__)[0]
- if not myDir: myDir = '.'
- replDict = {'testDir':'%s'%myDir}
- for i in self.config:
- try:
- self.config[i] = self.config.get(i)%(replDict)
- except:
- pass
- self.logger = TestingLogger(me.fileLogger)
- self.connection = me.database.connection()
- #self.connection = psycopg2.connect(me.dsn)
-
- self.testConfig = configurationManager.Config([('t','testPath', True, './TEST-BUGZILLA', ''),
- ('f','testFileName', True, 'lastrun.pickle', ''),
- ('', 'daysIntoPast', True, 0),])
- self.testConfig["persistentDataPathname"] = os.path.join(self.testConfig.testPath, self.testConfig.testFileName)
-
- def tearDown(self):
- self.logger.clear()
-
- def test_bugzilla_iterator(self):
- csv = ['bug_id,"bug_status","resolution","short_desc","cf_crash_signature"\n',
- '1,"RESOLVED",,"this is a comment","This sig, while bogus, has a ] bracket"',
- '2,"CLOSED","WONTFIX","comments are not too important","single [@ BogusClass::bogus_sig (const char**) ] signature"',
- '3,"ASSIGNED",,"this is a comment. [@ nanojit::LIns::isTramp()]","[@ js3250.dll@0x6cb96] [@ valid.sig@0x333333]"',
- '4,"CLOSED","RESOLVED","two sigs enter, one sig leaves","[@ layers::Push@0x123456] [@ layers::Push@0x123456]"',
- '5,"ASSIGNED","INCOMPLETE",,"[@ MWSBAR.DLL@0x2589f] and a broken one [@ sadTrombone.DLL@0xb4s455"',
- '6,"ASSIGNED",,"empty crash sigs should not throw errors",""',
- '7,"CLOSED",,"gt 525355 gt","[@gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)]"',
- '8,"CLOSED","RESOLVED","newlines in sigs","[@ legitimate(sig)] \n junk \n [@ another::legitimate(sig) ]"'
- ]
- correct = [ (1, "RESOLVED", "", "this is a comment", set([])),
- (2, "CLOSED", "WONTFIX", "comments are not too important", set(["BogusClass::bogus_sig (const char**)"])),
- (3, "ASSIGNED", "", "this is a comment. [@ nanojit::LIns::isTramp()]",
- set(["js3250.dll@0x6cb96", "valid.sig@0x333333"])),
- (4, "CLOSED", "RESOLVED", "two sigs enter, one sig leaves", set(["layers::Push@0x123456"])),
- (5, "ASSIGNED", "INCOMPLETE", "", set(["MWSBAR.DLL@0x2589f"])),
- (6, "ASSIGNED", "", "empty crash sigs should not throw errors", set([])),
- (7, "CLOSED", "", "gt 525355 gt", set(["gfx::font(nsTArray<nsRefPtr<FontEntry> > const&)"])),
- (8, "CLOSED", "RESOLVED", "newlines in sigs", set(['another::legitimate(sig)', 'legitimate(sig)']))
- ]
- for expected, actual in zip(bug.bugzilla_iterator(csv, iter), correct):
- assert expected == actual, "expected %s, got %s" % (str(expected), str(actual))
-
- def test_signature_is_found(self):
- global me
- assert bug.signature_is_found("js3250.dll@0x6cb96", me.cur)
- assert not bug.signature_is_found("sir_not_appearing_in_this_film", me.cur)
- me.cur.connection.rollback()
-
- def verify_tables(self, correct):
- global me
- # bug_status
- count = 0
- for expected, actual in zip(psy.execute(me.cur, "select id, status, resolution, short_desc from bugs order by 1"), correct["bugs"]):
- count += 1
- assert expected == actual, "expected %s, got %s" % (str(expected), str(actual))
- assert len(correct["bugs"]) == count, "expected %d entries in bugs but found %d" % (len(correct["bugs"]), count)
- #bug_associations
- count = 0
- for expected, actual in zip(psy.execute(me.cur, "select signature, bug_id from bug_associations order by 1, 2"), correct["bug_associations"]):
- count += 1
- assert expected == actual, "expected %s, got %s" % (str(expected), str(actual))
- assert len(correct["bug_associations"]) == count, "expected %d entries in bug_associations but found %d" % (len(correct["bug_associations"]), count)
-
- def test_insert_or_update_bug_in_database(self):
- #bugId, statusFromBugzilla, resolutionFromBugzilla, signatureListFromBugzilla
- #new * * empty
- #new * * 1 new
- #new * * 2 new
- #old * * empty
- #old new new
- global me
-
- def true(x, y):
- return True
-
- def hasYES(x, y):
- return "YES" in x
-
- me.cur = me.conn.cursor()
- #me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
- #me.cur.setLogger(me.fileLogger)
-
- psy.execute(me.cur, "delete from bug_status")
- me.cur.connection.commit()
-
- # test intial insert
- sample1 = [ (2,"CLOSED","WONTFIX","a short desc",set(["aaaa"])),
- (3,"NEW","","a short desc",set([])),
- (343324,"ASSIGNED","","a short desc",set(["bbbb","cccc"])),
- (343325,"CLOSED","RESOLVED","a short desc",set(["dddd"])),
- ]
- correct1 = { "bugs": [(2, "CLOSED", "WONTFIX", "a short desc"),
- (343324,"ASSIGNED","", "a short desc"),
- (343325,"CLOSED", "RESOLVED","a short desc")],
- "bug_associations": [("aaaa", 2),
- ("bbbb", 343324),
- ("cccc", 343324),
- ("dddd", 343325)]
- }
- for bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla in sample1:
- bug.insert_or_update_bug_in_database(bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla, me.cur, true)
- self.verify_tables(correct1)
-
- #test removing existing associations
- sample2 = [ (2,"CLOSED","WONTFIX","a short desc",set([])),
- (343324,"ASSIGNED","","a short desc",set(["bbbb"])),
- ]
- correct2 = { "bugs": [(343324,"ASSIGNED","","a short desc"),
- (343325,"CLOSED", "RESOLVED","a short desc")],
- "bug_associations": [("bbbb", 343324),
- ("dddd", 343325)]
- }
- for bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla in sample2:
- bug.insert_or_update_bug_in_database(bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla, me.cur, true)
- self.verify_tables(correct2)
-
- #test updating existing associations
- sample2 = [(343324,"CLOSED","RESOLVED","a short desc",set(["bbbb"])),
- ]
- correct2 = { "bugs": [(343324,"CLOSED","RESOLVED","a short desc"),
- (343325,"CLOSED", "RESOLVED","a short desc")],
- "bug_associations": [("bbbb", 343324),
- ("dddd", 343325)]
- }
- for bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla in sample2:
- bug.insert_or_update_bug_in_database(bugId, statusFromBugzilla, resolutionFromBugzilla, shortDescFromBugzilla, signatureListFromBugzilla, me.cur, true)
- self.verify_tables(correct2)
-
- def test_get_and_set_last_run_date(self):
- try:
- os.makedirs(self.testConfig.testPath)
- except OSError, x:
- if errno.EEXIST == x.errno: pass
- else: raise
- try:
- os.unlink(self.testConfig.persistentDataPathname)
- except OSError, x:
- pass
- def a_fixed_date():
- return dt.datetime(2009,05,04,15,10, tzinfo=UTC)
- assert bug.get_last_run_date(self.testConfig, a_fixed_date) == dt.datetime(2009, 4, 4, 15, 10, tzinfo=UTC)
- def another_fixed_date():
- return dt.datetime(2009,06,14,15,10, tzinfo=UTC)
- bug.save_last_run_date(self.testConfig, another_fixed_date)
- assert bug.get_last_run_date(self.testConfig) == another_fixed_date()
- try:
- os.unlink(self.testConfig.persistentDataPathname)
- except OSError, x:
- assert False, 'should have been able to delete the file: %s' % self.testConfig.persistentDataPathname
- os.rmdir(self.testConfig.testPath)
-
-
-if __name__ == "__main__":
- unittest.main()
View
119 socorro/unittest/cron/testDailyMatviews.py
@@ -1,119 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import datetime
-import unittest
-import psycopg2
-import socorro.cron.dailyMatviews # needed for Mock
-import socorro.cron.dailyMatviews as dailyMatviews
-
-from socorro.lib.datetimeutil import utc_now
-
-from socorro.unittest.config.commonconfig import databaseHost
-from socorro.unittest.config.commonconfig import databaseName
-from socorro.unittest.config.commonconfig import databaseUserName
-from socorro.unittest.config.commonconfig import databasePassword
-
-from mock import patch
-
-
-class mock_connection:
- def __init__(self, c):
- self.c = c
-
- def _void(self):
- pass
-
- commit = rollback = _void
-
- def cursor(self):
- return self.c
-
-
-class mock_psycopg2:
- InternalError = Exception
-
- def __init__(self, cursor):
- self.cursor = cursor
-
- def connect(self, *args, **kwargs):
- return mock_connection(self.cursor)
-
-
-class mock_cursor:
- def __init__(self, returns):
- self.returns = returns
- self.called = []
-
- def callproc(self, name, params):
- self.name = name
- self.called.append(name)
-
- def fetchone(self):
- if self.name in self.returns:
- return self.returns[self.name]
- return (True,)
-
- def execute(self, sql, params):
- pass
-
-
-class TestCase(unittest.TestCase):
- def setUp(self):
- self.config = {
- 'databaseHost': '',
- 'databasePassword': '',
- 'databaseName': '',
- 'databaseUserName': '',
- }
-
- def test_failing__update_product_versions(self):
- cursor = mock_cursor({
- 'update_product_versions': (False,),
- })
- dailyMatviews.psycopg2 = mock_psycopg2(cursor)
- with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
- dailyMatviews.update(self.config, 'some date')
- self.assertEqual(cursor.called, ['update_product_versions',
- 'update_signatures', 'update_os_versions', 'update_tcbs',
- 'update_adu', 'update_rank_compare', 'update_nightly_builds',
- 'update_build_adu', 'update_crashes_by_user',
- 'update_crashes_by_user_build', 'update_correlations',
- 'update_home_page_graph', 'update_home_page_graph_build',
- 'update_tcbs_build', 'update_explosiveness'])
- self.assertEqual(mock_logger.info.call_count, 15)
- self.assertEqual(mock_logger.warn.call_count, 1)
- self.assertEqual(mock_logger.error.call_count, 0)
-
- def test_all_works(self):
- cursor = mock_cursor({})
- dailyMatviews.psycopg2 = mock_psycopg2(cursor)
- with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
- dailyMatviews.update(self.config, 'some date')
- self.assertEqual(mock_logger.info.call_count, 15)
- self.assertEqual(mock_logger.warn.call_count, 0)
- self.assertEqual(mock_logger.error.call_count, 0)
-
- def test_mock_internal_error(self):
- cursor = mock_cursor({
- 'update_signatures': psycopg2.InternalError,
- })
- dailyMatviews.psycopg2 = mock_psycopg2(cursor)
- with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
- dailyMatviews.update(self.config, 'some date')
- self.assertEqual(mock_logger.info.call_count, 15)
- self.assertEqual(mock_logger.warn.call_count, 0)
- self.assertEqual(mock_logger.error.call_count, 1)
-
- # failure is OK for some functions
- def test_failing__update_adu(self):
- cursor = mock_cursor({
- 'update_adu': (False,),
- })
- dailyMatviews.psycopg2 = mock_psycopg2(cursor)
- with patch('socorro.cron.dailyMatviews.logger') as mock_logger:
- dailyMatviews.update(self.config, 'some date')
- self.assertEqual(mock_logger.info.call_count, 15)
- self.assertEqual(mock_logger.warn.call_count, 0)
- self.assertEqual(mock_logger.error.call_count, 0)
View
208 socorro/unittest/cron/testFtpScraper.py
@@ -1,208 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import errno
-import logging
-import os
-
-import psycopg2
-
-import unittest
-import socorro.lib.buildutil as buildutil
-import socorro.lib.ConfigurationManager as cfgManager
-import socorro.lib.util
-import socorro.lib.psycopghelper as psy
-import socorro.cron.ftpscraper as ftpscraper
-from socorro.unittest.testlib.loggerForTest import TestingLogger
-from socorro.unittest.testlib.testDB import TestDB
-import socorro.unittest.testlib.expectations as exp
-import cronTestconfig as testConfig
-
-
-class Me:
- """
- Initialize once per module
- """
- pass
-me = None
-
-
-def setup_module():
- global me
- if me:
- return
- me = Me()
- me.config = cfgManager.newConfiguration(configurationModule=testConfig,
- applicationName='Testing ftpscraper')
- myDir = os.path.split(__file__)[0]
- if not myDir:
- myDir = '.'
- replDict = {'testDir': '%s' % myDir}
- for i in me.config:
- try:
- me.config[i] = me.config.get(i) % (replDict)
- except:
- pass
- me.logFilePathname = me.config.logFilePathname
- if not me.logFilePathname:
- me.logFilePathname = 'logs/ftpscraper_test.log'
- logFileDir = os.path.split(me.logFilePathname)[0]
- try:
- os.makedirs(logFileDir)
- except OSError, x:
- if errno.EEXIST == x.errno:
- pass
- else:
- raise
- fileLog = logging.FileHandler(me.logFilePathname, 'a')
- fileLog.setLevel(logging.DEBUG)
- fileLogFormatter = logging.Formatter(
- '%(asctime)s %(levelname)s - %(message)s')
- fileLog.setFormatter(fileLogFormatter)
- me.fileLogger = logging.getLogger("ftpscraper")
- me.fileLogger.addHandler(fileLog)
- # Got trouble? See what's happening by uncommenting the next three lines
- #stderrLog = logging.StreamHandler()
- #stderrLog.setLevel(10)
- #me.fileLogger.addHandler(stderrLog)
-
- me.dsn = "host=%s dbname=%s user=%s password=%s" % \
- (me.config.databaseHost, me.config.databaseName,
- me.config.databaseUserName, me.config.databasePassword)
- me.testDB = TestDB()
- me.testDB.removeDB(me.config, me.fileLogger)
- me.testDB.createDB(me.config, me.fileLogger)
- try:
- me.conn = psycopg2.connect(me.dsn)
- me.cur = me.conn.cursor(cursor_factory=psy.LoggingCursor)
- except Exception, x:
- print "Exception in setup_module() connecting to db: ", type(x), x
- socorro.lib.util.reportExceptionAndAbort(me.fileLogger)
-
-
-def teardown_module():
- global me
- me.testDB.removeDB(me.config, me.fileLogger)
- me.conn.close()
- try:
- os.unlink(me.logFilePathname)
- except:
- pass
-
-
-class TestFtpScraper(unittest.TestCase):
- def setUp(self):
- global me
- self.config = cfgManager.newConfiguration(
- configurationModule=testConfig,
- applicationName='Testing ftpscraper')
-
- myDir = os.path.split(__file__)[0]
- if not myDir:
- myDir = '.'
- replDict = {'testDir': '%s' % myDir}
- for i in self.config:
- try:
- self.config[i] = self.config.get(i) % (replDict)
- except:
- pass
- self.logger = TestingLogger(me.fileLogger)
-
- self.testConfig = cfgManager.Config([('t', 'testPath',
- True, './TEST-BUILDS', ''),
- ('f', 'testFileName',
- True, 'lastrun.pickle', '')])
- self.testConfig["persistentDataPathname"] = os.path.join(
- self.testConfig.testPath, self.testConfig.testFileName)
-
- def tearDown(self):
- self.logger.clear()
-
- def test_getLinks(self):
- self.config.products = ('PRODUCT1', 'PRODUCT2')
- self.config.base_url = 'http://www.example.com/'
-
- fake_response_url = "%s%s" % (self.config.base_url,
- self.config.products[0])
- fake_response_contents = """
- blahblahblahblahblah
- <a href="product1-v1.en-US.p1.txt">product1-v1.en-US.p1.txt</a>
- <a href="product1-v1.en-US.p1.zip">product1-v1.en-US.p1.zip</a>
- <a href="product2-v2.en-US.p2.txt">product2-v2.en-US.p2.txt</a>
- <a href="product2-v2.en-US.p2.zip">product2-v2.en-US.p2.zip</a>
- blahblahblahblahblah
- """
-
- fakeResponse = exp.DummyObjectWithExpectations()
- fakeResponse.code = 200
- fakeResponse.expect('read', (), {}, fake_response_contents)
- fakeResponse.expect('close', (), {})
-
- fakeUrllib2 = exp.DummyObjectWithExpectations()
- fakeUrllib2.expect('urlopen', (fake_response_url,), {}, fakeResponse)
-
- actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1',
- startswith='product1', urllib=fakeUrllib2)
- expected = ['product1-v1.en-US.p1.txt',
- 'product1-v1.en-US.p1.zip']
- assert actual == expected, "expected %s, got %s" % (expected, actual)
-
- fakeResponse = exp.DummyObjectWithExpectations()
- fakeResponse.code = 200
- fakeResponse.expect('read', (), {}, fake_response_contents)
- fakeResponse.expect('close', (), {})
-
- fakeUrllib2 = exp.DummyObjectWithExpectations()
- fakeUrllib2.expect('urlopen', (fake_response_url,), {}, fakeResponse)
-
- expected = ['product1-v1.en-US.p1.zip',
- 'product2-v2.en-US.p2.zip']
- actual = ftpscraper.getLinks('http://www.example.com/PRODUCT1',
- endswith='.zip', urllib=fakeUrllib2)
- assert actual == expected, "expected %s, got %s" % (expected, actual)
-
- def test_parseInfoFile(self):
- self.config.products = ('PRODUCT1', 'PRODUCT2')
- self.config.base_url = 'http://www.example.com/'
-
- fake_response_url = "%s%s" % (self.config.base_url,
- self.config.products[0])
- fake_response_contents = """
- 20111011042016
- http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663
- """
-
- fakeResponse = exp.DummyObjectWithExpectations()
- fakeResponse.code = 200
- fakeResponse.expect('read', (), {}, fake_response_contents)
- fakeResponse.expect('close', (), {})
-
- fakeUrllib2 = exp.DummyObjectWithExpectations()
- fakeUrllib2.expect('urlopen', (fake_response_url,), {}, fakeResponse)
-
- rev = 'http://hg.mozilla.org/releases/mozilla-aurora/rev/327f5fdae663'
- expected = {
- 'buildID': '20111011042016',
- 'rev': rev
- }
- actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1',
- nightly=True, urllib=fakeUrllib2)
- assert actual == expected, "expected %s, got %s" % (expected, actual)
-
- fake_response_contents = """
- buildID=20110705195857
- """
-
- fakeResponse = exp.DummyObjectWithExpectations()
- fakeResponse.code = 200
- fakeResponse.expect('read', (), {}, fake_response_contents)
- fakeResponse.expect('close', (), {})
-
- fakeUrllib2 = exp.DummyObjectWithExpectations()
- fakeUrllib2.expect('urlopen', (fake_response_url,), {}, fakeResponse)
- expected = {'buildID': '20110705195857'}
- actual = ftpscraper.parseInfoFile('http://www.example.com/PRODUCT1',
- nightly=False, urllib=fakeUrllib2)
- assert actual == expected, "expected %s, got %s" % (expected, actual)

0 comments on commit ce5026c

Please sign in to comment.
Something went wrong with that request. Please try again.