Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

bring docs back home

  • Loading branch information...
commit 2f0f714995cc04e2f76c01b8ab76198fb33c8f31 1 parent a8b4020
@rhelmer rhelmer authored
Showing with 6,025 additions and 77 deletions.
  1. +1 −0  .gitignore
  2. +4 −4 README
  3. +130 −0 docs/Makefile
  4. BIN  docs/ProcessedDumpStorage.1.0.png
  5. +0 −8 docs/README.txt
  6. BIN  docs/Socorro.directory.diagram.2.2.png
  7. BIN  docs/SocorroSchema.Aggregate.20090722.png
  8. BIN  docs/SocorroSchema.CrashData.20090722.png
  9. BIN  docs/SocorroSchema_2_0.png
  10. +369 −0 docs/codeanddatabaseupdate.rst
  11. +37 −0 docs/codingconventions.rst
  12. +55 −0 docs/collector.rst
  13. +132 −0 docs/commonconfig.rst
  14. +216 −0 docs/conf.py
  15. +87 −0 docs/crashmover.rst
  16. +62 −0 docs/crashreprofilteringreport.rst
  17. +36 −0 docs/databaseadminfunctions.rst
  18. +374 −0 docs/databaseschema.rst
  19. +107 −0 docs/databasesetup.rst
  20. +136 −0 docs/databasetablesbysource.rst
  21. +116 −0 docs/deferredcleanup.rst
  22. +22 −0 docs/deferredjobstorage.rst
  23. +25 −0 docs/deployment.rst
  24. +27 −0 docs/development.rst
  25. +183 −0 docs/diskperformancetests.rst
  26. +89 −0 docs/dumpingdumptables.rst
  27. +12 −0 docs/filesystem.rst
  28. +112 −0 docs/glossary.rst
  29. +45 −0 docs/index.rst
  30. +123 −65 INSTALL → docs/installation.rst
  31. +173 −0 docs/jsondumpstorage.rst
  32. +170 −0 docs/make.bat
  33. +158 −0 docs/middleware.rst
  34. +173 −0 docs/monitor.rst
  35. +62 −0 docs/overview.rst
  36. +29 −0 docs/package.rst
  37. +87 −0 docs/processeddumpstorage.rst
  38. +19 −0 docs/processor.rst
  39. +107 −0 docs/reportdatabasedesign.rst
  40. +12 −0 docs/reporter.rst
  41. +40 −0 docs/requirements.rst
  42. BIN  docs/schema.original.png
  43. BIN  docs/schema.partitioned.001.png
  44. +427 −0 docs/schema.rst
  45. +26 −0 docs/server.rst
  46. +90 −0 docs/signaturegeneration.rst
  47. +70 −0 docs/standalone.rst
  48. +21 −0 docs/standardjobstorage.rst
  49. +64 −0 docs/throttling.rst
  50. +32 −0 docs/tools/rstrip.py
  51. +49 −0 docs/topcrashersbysignature.rst
  52. +113 −0 docs/topcrashersbyurl.rst
  53. +332 −0 docs/ui.rst
  54. +204 −0 docs/uiinstallation.rst
  55. +39 −0 docs/uitroubleshooting.rst
  56. +86 −0 docs/unittesting.rst
  57. +869 −0 docs/upgrade.rst
  58. +73 −0 docs/writingdocs.rst
View
1  .gitignore
@@ -13,3 +13,4 @@ socorro-virtualenv
socorro/unittest/database/logs/
socorro/unittest/testlib/logs/
stackwalk/
+docs/_build/
View
8 README
@@ -1,9 +1,8 @@
Socorro
A server for collecting, processing, and displaying crash reports from
-clients using the Breakpad libraries.
+clients using the Breakpad libraries - http://code.google.com/p/google-breakpad/
-https://github.com/mozilla/socorro
-http://code.google.com/p/google-breakpad/
+See http://socorro.readthedocs.org/en/latest/installation.html for installation instructions.
Socorro mailing list:
https://lists.mozilla.org/listinfo/tools-socorro
@@ -11,4 +10,5 @@ https://lists.mozilla.org/listinfo/tools-socorro
Socorro/Breakpad IRC channel:
irc://irc.mozilla.org/breakpad
-See the INSTALL file for installation instructions.
+You can find the Socorro documentation online at http://socorro.readthedocs.org
+The documentation source is in ./docs/
View
130 docs/Makefile
@@ -0,0 +1,130 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Socorro.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Socorro.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/Socorro"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Socorro"
+ @echo "# devhelp"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ make -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
View
BIN  docs/ProcessedDumpStorage.1.0.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
8 docs/README.txt
@@ -1,8 +0,0 @@
-The Socorro documentation is in a separate module:
-https://github.com/mozilla/socorro-docs
-
-The docs can be read online using Read The Docs:
-http://socorro.readthedocs.org
-
-If you are interested in writing docs please see:
-http://socorro.readthedocs.org/en/latest/writingdocs.html
View
BIN  docs/Socorro.directory.diagram.2.2.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN  docs/SocorroSchema.Aggregate.20090722.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN  docs/SocorroSchema.CrashData.20090722.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN  docs/SocorroSchema_2_0.png
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
369 docs/codeanddatabaseupdate.rst
@@ -0,0 +1,369 @@
+.. index:: codeanddatabaseupdate
+
+.. _codeanddatabaseupdate-chapter:
+
+
+Code and Database Update
+========================
+
+Socorro Wish List
+-----------------
+
+One of my (griswolf) directives is approximately "make everything work
+efficiently and the same." Toward this end, there are several tasks:
+
+Probably most important, we have an inefficient database design, and
+some inefficient code working with it.
+
+Next, we have a collection of 'one-off' code (and database schemas)
+that could be more easily maintained using a common infrastructure,
+common coding conventions, common schema layout, common patterns.
+
+Finally, we have enhancement requests that would become more feasible
+after such changes: Such requests would be more easily handled in a
+cleaner programming environment; and in a cleaner environment there
+might be fewer significant bugs, leaving more time to work on
+enhancements.
+
+Current state: See [[SocorroDatabaseSchema]]
+
+
+Another Way to do Materialized Views?
+-------------------------------------
+
+The current system is somewhere between ad hoc reporting and a star
+architecture. The main part of this proposal focuses on converting
+further toward a star architecture. However there may be another way:
+MapReduce techniques, which could possibly be run external to Mozilla
+(for instance: Amazon Web Services) could be used to mine dump files
+and create statistical data stored in files or database. Lars
+mentioned to me that we now have some statistics folk on board who are
+interested in this.
+
+
+Database Design
+---------------
+* There are some legacy tables (reports, topcrasher) that are not normalized. Other tables are partly normalized. Non-normal has consequences:
+ * Data is duplicated, causing possible synchronization issues.
+ * JOSH: duplicated data is normal for materialized views and is not a problem a priori.
+ * Data is duplicated, increasing size.
+ * JOSH: I don't believe that the matview tables are that large, although we will want to look at partitioning them in the future because they will continue to grow.
+ * FRANK: Lars points out that size-limiting partitions which reference each other must all be partitioned on the same key. This makes partitions a little more interesting
+ * SELECT statements on multiple varchar fields, even when indexed, are probably slower than SELECT statements on a single foreign key. (And even if not, maintaining larger index tables has a time and space cost)
+* There are legacy tables that contain deprecated columns, a slight inefficiency.
+* In some cases, separable details are conflated, making it difficult to access by a single area of concern. For instance, the table that describes our products has an os_name column, requiring us to pretend we deal with an os named 'ALL' in order to examine product data without regard to os.
+* According to Postgresql consultants, some types are not as efficient as others. Example TEXT (which we use only a little) is slightly more time-efficient than VARCHAR(n) (which we mostly use)
+ * JOSH: this is a minor issue, and should only be changed if we're modifying the fields/tables anyway.
+ * FRANK: We have already run into a size limitation for signatures which are now VARCHAR(255). Experiment shows that conversion to TEXT is slow because of index rebuilding, but conversion to VARCHAR(BIGGER_NUMBER) can be done by manipulating typemod (the number of chars in VARCHAR) in the system tables. So change from VARCHAR to TEXT needs to be scheduled in advance, with an expected 'long' turn around.
+* Current indexes were carefully audited during PGExperts week. Schema changes will require careful reevaluation
+
+Commonality
+-----------
+
+* Some of the tables that provide statistics (Mean Time Before Failure, for example) use a variant of the "Star" data warehousing pattern, which is well known and understood. Some do not. After discussion we have reached agreement that all should be partly 'starred'
+ * osdims and productdims are appropriate dimension tables for each view that cares about operating system or product
+ * url and signature 'dimension' tables are used to filter materialized views:
+ * the 'fact' tables for views will use ids from these filter/dimension tables
+ * the filter/dimension tables will hold only data that has passed a particular frequency threshold, initial guess at threshold: 3 per week.
+* Python code has been written by a variety of people with various skill levels, doing things in a variety of ways. Mostly, this is acceptable, but required changes give us an opportunity.
+* We now specify Python version 2.4, which is adequate. Possible to upgrade to 2.5.x or 2.6.x with both ease and safety. This is an opportunity to do so. No code needs to change for this.
+* New features (safely) available in Python 2.5:
+ * unified try/except/finally: instead of a try/finally block holding a try/except block
+ * there is a very nice with: syntax useful for block-scoped non GC'd resources such as open files (like try: with an automatic finally: at block end)
+ * generators are significantly more powerful, which might have some uses in our code
+ * and lots more that seems less obviously useful to Socorro
+ * better exception hierarchy
+* New features (safely) available in Python 2.6
+ * json library ships with Python 2.6
+ * multiprocessing library parallel to threading library ships with Python 2.6
+ * Command line option '-3' flags things that will work differently or fail in Python 3 (looking ahead is good)
+* We use nosetests which is not correctly and fully functional in a Python 2.4 environment.
+
+Viewable Interface
+------------------
+
+* We have been gradually providing a more useful view of the crash data. Sometimes this is intrinsically hard, sometimes it is made more difficult by our schema.
+* We have requests for:
+ * Better linkage between crash reports and bugs
+ * Ability to view by OS and OS version, by signature, by product, by product version (some of this will be easier with a new schema)
+ * Ability to view historical data, current data, (sliding) windows of data and trends
+* Some of the requests seem likely to be too time or space costly. In some cases these might be feasible with a more efficient system
+
+Consequences of Possible Changes
+--------------------------------
+
+ * (Only) Add new tables (two kinds of changes)
+ * "replace in place", for instance add table reports_normal while leaving table reports in place)
+ * "brand new", for instance add new productdims and osdims tables to serve a new tobcrashbysignature table
+ * Existing views are not impacted (for good or ill)
+ * Duplication of data (some tables near normal form, some not, etc) becomes worse than it now is
+ * No immediate need to migrate data: Options
+ * Maybe provide two views: "Historic" and "Current"
+ * Maybe write 'orrible look-both-ways code to access both tables from single view
+ * Maybe migrate data
+ * Code that looks at old schema is (mostly?) unchanged
+ * Code that looks at new schema is opportunity for improved design, etc.
+ * Can do one thing at a time, with multiple 'easy' rollouts (each one is still a rollout, though)
+ * Long term goal: Stop using old tables and code
+* (Only) Drop redundant or deprecated columns in existing tables:
+ * Existing views are no less useful, Viewer and Controller code will need some maintenance
+ * Data migration is 'simple'
+ * beware that dropped columns may be part of a (foreign) key or index
+ * Data migration is needed at rollout
+ * Minimally useful
+* Optimize database types, indexes, keys:
+ * Existing views are not much impacted
+ * May want to optimize queries in Viewer and Controller code
+ * May need to guard for field size or type in Controller code
+ * Details of changes are 'picky' and may need some hand holding by consultants, maybe testing.
+* Normalize existing tables (while adding new tables as needed):
+ * Much existing code needs re-write
+ * With different Model comes a need for different Viewers and Controllers
+ * Opportunity to clarify old code
+ * Opportunity to optimize queries
+ * Data migration is needed at rollout
+ * Rollout is complex (but need only one for complete conversion)
+ * JOSH: in general, Matview generation should be optimized to be insert-only. In some cases, this will involve having a "current week" partition which gets dropped and recreated until the current week is completed. Updates are generally at least 4x as expensive as inserts.
+
+Rough plan as of 2009 June
+--------------------------
+
+* Soon: Materialized views will make use of dimensions and 'filtered dimensions' tables
+* Later: Normalize the 'raw' data to make use of tables describing operating system and product details. Leave signatures and urls raw
+
+Specific Database Changes
+-------------------------
+
+**Star Data Warhousing**
+
+**Existing tables**
+
+* --dimension: signaturedims: associate the base crash signature
+ string with an id-- Use signature TEXT directly
+* dimension: productdims: associate a product, version, release and os_name with an id
+ * os_name is neither sufficient for os drill-down (which wants os_version) nor properly part of a product dimension
+* dimension: urldims: associate (a large number of) domains and urls, each pair with an id
+* config: mtbfconfig: specifies the date-interval during which a given product (productdims) is of interest for MTBF analysis
+* config: tcbyurlconfig: specifies whether a particular product (productdims) is now of interest for Top Crash by URL analysis.
+* fact: mtbffacts: collects daily summary of average time before failure for each product
+* --report: topcrashurlfactsreports: associates a crash uuid and a
+ comment with a row of topcrashurlfacts ?Apparently never used?--
+
+**Needed/Changed tables**
+
+**Matview changes "Soon"**
+
+* config (new): product_visibility: Specifies date interval during which a product (productdims id) is of interest for any view. ?Replaces mtbfconfig?
+* dimension (new): osdims: associate an os name and os version with an id
+* dimension (edit): productdims: remove the os_name column (replaced by another dimension osdims above)
+* fact (replace): topcrashers: The table now in use to provide Top Crash by Signature view. Will be replaced by topcrashfacts
+* fact (new): topcrashfacts: collect periodic count of crashes, average uptime before crash and rank of each signature by signature, os, product
+ * replaces existing topcrashers table which is poorly organized for current needs
+* config (new): tcbysignatureconfig: specify which products and operating systems are currently of interest for tcbysigfacts
+* fact: (renamed, edit) top_crashes_by_url: collects daily summary of crashes by product, url (productdims, urldims)
+* fact: (new): top_crashes_by_url_signature: associates a given row from top_crashes_by_url with one or more signatures
+
+**Incoming (raw) changes "Later"**
+
+* details (new): osdetails, parallel to osdims, but on the incoming side will be implemented later
+* details (new): productdetails, parallel to productdims, but on the incoming side will be implemented later
+* reports: Holds details of each analyzed crash report. It is not in normal form, which causes some ongoing difficulty
+ * columns product, version, build should be replaced by productdetails foreign key later
+ * column signature LARS: NULL is a legal value here. We'll have to make sure that we use left outer joins to retrieve the report records.
+ * columns cpu_name, cpu_info are not currently in use in any other table, but could be a foreign key into cpudims
+ * columns os_name, os_version should be replaced by osdims foreign key
+ * columns email, user_id are deprecated and should be dropped
+
+**Details**
+
+**New or significantly changed tables**
+
+New product_visibility table (soon, matview)::
+
+ table product_visibility (
+ id serial NOT NULL PRIMARY KEY,
+ productdims_id integer not null,
+ start_date timestamp, -- used by MTBF
+ end_date timestamp,
+ ignore boolean default False -- force aggregation off for this product id
+
+New osdims table (soon, matview) NOTE: Data available only if
+'recently frequent'::
+
+ table osdims(
+ id serial NOT NULL PRIMARY KEY,
+ os_name TEXT NOT NULL,
+ os_version TEXT);
+ constraint osdims_key (os_name, os_version) unique (os_name, os_version);
+
+Edited productdims table (soon, matview) NOTE: use case for adding
+products is under discussion::
+
+ CREATE TYPE release_enum AS ENUM ('major', 'milestone', 'development');
+ table productdims (
+ id serial NOT NULL PRIMARY KEY,
+ product TEXT NOT NULL,
+ version TEXT NOT NULL,
+ release release_enum NOT NULL,
+ constraint productdims_key (product, version) unique ( product, version )
+ );
+
+New product_details table (later, raw data) NOTE: All data will be
+stored (raw data should not lose details)::
+
+ table product_details (
+ id serial NOT NULL PRIMARY KEY,
+ product TEXT NOT NULL, -- /was/ character varying(30)
+ version TEXT NOT NULL, -- /was/ character varying(16)
+ release release_enum NOT NULL -- /was/ character varying(50) NOT NULL
+ );
+
+Edit mtbffacts to use edited productdims and new osdims (soon,
+matview)::
+
+ table mtbffacts (
+ id serial NOT NULL PRIMARY KEY,
+ avg_seconds integer NOT NULL,
+ report_count integer NOT NULL,
+ window_end timestamp, -- was DATE
+ productdims_id integer,
+ osdims_id integer
+ constraint mtbffacts_key unique ( productdims_id, osdims_id, day );
+ );
+
+New top_crashes_by_signature table (soon, matview)::
+
+ table top_crashes_by_signature (
+ id serial NOT NULL PRIMARY KEY,
+ count integer NOT NULL DEFAULT 0,
+ average_uptime real DEFAULT 0.0,
+ window_end timestamp without time zone,
+ window_size interval,
+ productdims_id integer NOT NULL, -- foreign key. NOTE: Filtered by recent frequency
+ osdims_id integer NOT NULL, -- foreign key. NOTE: Filtered by recent frequency
+ signature TEXT
+ constraint top_crash_by_signature_key (window_end, signature, productdims_id, osdims_id) unique (window_end, signature, productdims_id, osdims_id)
+ );
+ -- some INDEXes are surely needed --
+
+New/Renamed top_crashes_by_url table (soon, matview)::
+
+ table top_crashes_by_url (
+ id serial NOT NULL,
+ count integer NOT NULL,
+ window_end timestamp without time zone NOT NULL,
+ window_size interval not null,
+ productdims_id integer,
+ osdims_id integer NOT NULL,
+ urldims_id integer
+ constraint top_crashes_by_url_key (uridims_id,osdims_id,productdims_id, window_end) unique (uridims_id,osdims_id,productdims_id, window_end)
+ );
+
+New top_crashes_by_url_signature (soon, matview)::
+
+ table top_crash_by_url_signature (
+ top_crashes_by_url_id integer, -- foreign key
+ count integer NOT NULL,
+ signature TEXT NOT NULL
+ constraint top_crashes_by_url_signature_key (top_crashes_by_url_id,signature) unique (top_crashes_by_url_id,signature)
+ );
+
+New crash_reports table (later, raw view) Replaces reports table::
+
+ table crash_reports (
+ id serial NOT NULL PRIMARY KEY,
+ uuid TEXT NOT NULL -- /was/ character varying(50)
+ client_crash_date timestamp with time zone,
+ install_age integer,
+ last_crash integer,
+ uptime integer,
+ cpu_name TEXT, -- /was/ character varying(100),
+ cpu_info TEXT, -- /was/ character varying(100),
+ reason TEXT, -- /was/ character varying(255),
+ address TEXT, -- /was/ character varying(20),
+ build_date timestamp without time zone,
+ started_datetime timestamp without time zone,
+ completed_datetime timestamp without time zone,
+ date_processed timestamp without time zone,
+ success boolean,
+ truncated boolean,
+ processor_notes TEXT,
+ user_comments TEXT, -- /was/ character varying(1024),
+ app_notes TEXT, -- /was/ character varying(1024),
+ distributor TEXT, -- /was/ character varying(20),
+ distributor_version TEXT, -- /was/ character varying(20)
+ signature TEXT,
+ productdims_id INTEGER, -- /new/ foreign key NOTE Filtered by recent frequency
+ osdims_id INTEGER, -- /new/ foreign key NOTE Filtered by recent frequency
+ urldims_id INTEGER -- /new/ foreign key NOTE Filtered by recent frequency
+ -- /remove - see productdims_id/ - product character varying(30),
+ -- /remove - see productdims_id/ version character varying(16),
+ -- /remove - redundant with build_date/ -- build character varying(30),
+ -- /remove - see urldims_id/ url character varying(255),
+ -- /remove - see osdims_id/ os_name character varying(100),
+ -- /remove - see osdims_id/ os_version character varying(100),
+ -- /remove - deprecated/ email character varying(100),
+ -- /remove - deprecated/ user_id character varying(50),
+ );
+ -- This is a partitioned table: INDEXes are provided on date-based partitions
+
+Tables with Minor Changes: varchar->text::
+
+ table branches (
+ product TEXT NOT NULL, -- /was/ character varying(30)
+ version TEXT NOT NULL, -- /was/ character varying(16)
+ branch TEXT NOT NULL, -- /was/ character varying(24)
+ PRIMARY KEY (product, version)
+
+ table extensions (
+ report_id integer NOT NULL, -- foreign key
+ date_processed timestamp without time zone,
+ extension_key integer NOT NULL,
+ extension_id TEXT NOT NULL, -- /was/ character varying(100)
+ extension_version TEXT -- /was/ character varying(16)
+
+ table frames (
+ report_id integer NOT NULL,
+ date_processed timestamp without time zone,
+ frame_num INTEGER NOT NULL,
+ signature TEXT -- /was/ varchar(255)
+ );
+
+ table priority_jobs
+ uuid TEXT NOT NULL PRIMARY KEY -- /was/ varchar(255)
+
+ table processors (
+ id serial NOT NULL PRIMARY KEY,
+ name TEXT NOT NULL UNIQUE, -- /was/ varchar(255)
+ startdatetime timestamp without time zone NOT NULL,
+ lastseendatetime timestamp without time zone
+ );
+
+ table jobs (
+ id serial NOT NULL PRIMARY KEY,
+ pathname TEXT NOT NULL, -- /was/ character varying(1024)
+ uuid TEXT NOT NULL UNIQUE, -- /was/ varchar(50)
+ owner integer,
+ priority integer DEFAULT 0,
+ queueddatetime timestamp without time zone,
+ starteddatetime timestamp without time zone,
+ completeddatetime timestamp without time zone,
+ success boolean,
+ message TEXT,
+ FOREIGN KEY (owner) REFERENCES processors (id)
+ );
+
+ table urldims (
+ id serial NOT NULL PRIMARY KEY,
+ domain TEXT NOT NULL, -- /was/ character varying(255)
+ url TEXT NOT NULL -- /was/ character varying(255)
+ key url -- for drilling by url
+ key domain -- for drilling by domain
+ );
+
+ table topcrashurlfactsreports (
+ id serial NOT NULL PRIMARY KEY,
+ uuid TEXT NOT NULL, -- /was/ character varying(50)
+ comments TEXT, -- /was/ character varying(500)
+ topcrashurlfacts_id integer
+ );
View
37 docs/codingconventions.rst
@@ -0,0 +1,37 @@
+.. index:: codingconventions
+
+.. _codingconventions-chapter:
+
+
+Coding Conventions
+==================
+
+Introduction
+------------
+
+The following coding conventions are designed to ensure that the
+Socorro code is easy to read, hack, test, and deploy.
+
+Style Guide
+-----------
+
+* Python should follow PEP 8 with 4 space indents
+* PHP code follows the PEAR coding standard
+* JavaScript is indented by four spaces
+* :ref:`unittesting-chapter` is strongly encouraged
+
+Review
+------
+
+New checkins that are non-trivial should be reviewed by one of the
+core hackers. The commit message should indicate the reviewer and the
+issue number if applicable.
+
+Testing
+-------
+
+Any features that are only available to admins should be tested to
+ensure that only non-admin users to not have access.
+
+Before checking in changes to the socorro python code, be sure to run
+the unit tests.
View
55 docs/collector.rst
@@ -0,0 +1,55 @@
+.. index:: collector
+
+.. _collector-chapter:
+
+
+Collector
+=========
+
+Collector is an application that runs under Apache using mod-python.
+Its task is accepting crash reports from remote clients and saving
+them in a place and format usable by further applications.
+
+Raw crashes are accepted via HTTP POST. The form data from the POST is
+then arranged into a JSON and saved into the local file system. The
+collector is responsible for assigning an ooid? (Our Own ID) to the
+crash. It also assigns a Throttle? value which determines if the crash
+is eventually to go into the relational database.
+
+Should the saving to a local file system fail, there is a fallback
+storage mechanism. A second file system can be configured to take the
+failed saves. This file system would likely be an NFS mounted file
+system.
+
+After a crash is saved, there is an app called :ref:`crashmover-chapter` that
+will transfer the crashes to HBase.
+
+Collector Python Configuration
+------------------------------
+
+Like all the Socorro applications, the configuration is actually
+executable Python code. Two configuration files are relevant for
+collector
+
+* Copy ``.../scripts/config/commonconfig.py.dist`` to
+ `.../config/commonconfig.py`. This configuration file contains
+ constants used by many of the Socorro applications.
+* Copy ``.../scripts/config/collectorconfig.py.dist`` to
+ ``.../config/collectorconfig.py``
+
+Common Configuration
+--------------------
+
+There are two constants in '.../scripts/config/commonconfig.py' of
+interest to collector: `jsonFileSuffix`, and `dumpFileSuffix`. Other
+constants in this file are ignored.
+
+To setup the common configuration, see :ref:`commonconfig-chapter`.
+
+Collector Configuration
+-----------------------
+
+collectorconfig.py has several options to adjust how files are stored:
+
+`See sample config code on Github
+<https://github.com/mozilla/socorro/blob/master/scripts/config/collectorconfig.py.dist>`_
View
132 docs/commonconfig.rst
@@ -0,0 +1,132 @@
+.. index:: commonconfig
+
+.. _commonconfig-chapter:
+
+
+Common Config
+=============
+
+To avoid repetition between configurations of a half dozen
+independently running applications, common settings are consolidated
+in a common configuration file:
+``OB.../scripts/config/commonconfig.py.dist``.
+
+All Socorro applications have these constants available to them. For a
+Socorro applications that are command line driven, each of these
+default values can be overidden by a command line switch of the same
+name.
+
+To setup this configuration file, just copy the example,
+``.../scripts/config/commonconfig.py.dist`` to
+``.../scripts/config/commonconfig.py``.
+
+Edit the file for your local situation.::
+
+ import socorro.lib.ConfigurationManager as cm
+ import datetime
+ import stat
+
+ #---------------------------------------------------------------------------
+ # Relational Database Section
+
+ databaseHost = cm.Option()
+ databaseHost.doc = 'the hostname of the database servers'
+ databaseHost.default = 'localhost'
+
+ databasePort = cm.Option()
+ databasePort.doc = 'the port of the database on the host'
+ databasePort.default = 5432
+
+ databaseName = cm.Option()
+ databaseName.doc = 'the name of the database within the server'
+ databaseName.default = ''
+
+ databaseUserName = cm.Option()
+ databaseUserName.doc = 'the user name for the database servers'
+ databaseUserName.default = ''
+
+ databasePassword = cm.Option()
+ databasePassword.doc = 'the password for the database user'
+ databasePassword.default = ''
+
+ #---------------------------------------------------------------------------
+ # Crash storage system
+
+ jsonFileSuffix = cm.Option()
+ jsonFileSuffix.doc = 'the suffix used to identify a json file'
+ jsonFileSuffix.default = '.json'
+
+ dumpFileSuffix = cm.Option()
+ dumpFileSuffix.doc = 'the suffix used to identify a dump file'
+ dumpFileSuffix.default = '.dump'
+
+ #---------------------------------------------------------------------------
+ # HBase storage system
+
+ hbaseHost = cm.Option()
+ hbaseHost.doc = 'Hostname for hbase hadoop cluster. May be a VIP or load balancer'
+ hbaseHost.default = 'localhost'
+
+ hbasePort = cm.Option()
+ hbasePort.doc = 'hbase port number'
+ hbasePort.default = 9090
+
+ hbaseTimeout = cm.Option()
+ hbaseTimeout.doc = 'timeout in milliseconds for an HBase connection'
+ hbaseTimeout.default = 5000
+
+ #---------------------------------------------------------------------------
+ # misc
+
+ processorCheckInTime = cm.Option()
+ processorCheckInTime.doc = 'the time after which a processor is considered dead (hh:mm:ss)'
+ processorCheckInTime.default = "00:05:00"
+ processorCheckInTime.fromStringConverter = lambda x: str(cm.timeDeltaConverter(x))
+
+ startWindow = cm.Option()
+ startWindow.doc = 'The start of the single aggregation window (YYYY-MM-DD [hh:mm:ss])'
+ startWindow.fromStringConverter = cm.dateTimeConverter
+
+ deltaWindow = cm.Option()
+ deltaWindow.doc = 'The length of the single aggregation window ([dd:]hh:mm:ss)'
+ deltaWindow.fromStringConverter = cm.timeDeltaConverter
+
+ defaultDeltaWindow = cm.Option()
+ defaultDeltaWindow.doc = 'The length of the single aggregation window ([dd:]hh:mm:ss)'
+ defaultDeltaWindow.fromStringConverter = cm.timeDeltaConverter
+
+ # override this default for your particular cron task
+ defaultDeltaWindow.default = '00:12:00'
+
+ endWindow = cm.Option()
+ endWindow.doc = 'The end of the single aggregation window (YYYY-MM-DD [hh:mm:ss])'
+ endWindow.fromStringConverter = cm.dateTimeConverter
+
+ startDate = cm.Option()
+ startDate.doc = 'The start of the overall/outer aggregation window (YYYY-MM-DD [hh:mm])'
+ startDate.fromStringConverter = cm.dateTimeConverter
+
+ deltaDate = cm.Option()
+ deltaDate.doc = 'The length of the overall/outer aggregation window ([dd:]hh:mm:ss)'
+ deltaDate.fromStringConverter = cm.timeDeltaConverter
+
+ initialDeltaDate = cm.Option()
+ initialDeltaDate.doc = 'The length of the overall/outer aggregation window ([dd:]hh:mm:ss)'
+ initialDeltaDate.fromStringConverter = cm.timeDeltaConverter
+
+ # override this default for your particular cron task
+ initialDeltaDate.default = '4:00:00:00'
+
+ minutesPerSlot = cm.Option()
+ minutesPerSlot.doc = 'how many minutes per leaf directory in the date storage branch'
+ minutesPerSlot.default = 1
+
+ endDate = cm.Option()
+ endDate.doc = 'The end of the overall/outer aggregation window (YYYY-MM-DD [hh:mm:ss])'
+ endDate.fromStringConverter = cm.dateTimeConverter
+
+ debug = cm.Option()
+ debug.doc = 'do debug output and routines'
+ debug.default = False
+ debug.singleCharacter = 'D'
+ debug.fromStringConverter = cm.booleanConverter
View
216 docs/conf.py
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+#
+# Socorro documentation build configuration file, created by
+# sphinx-quickstart on Wed Sep 21 14:59:08 2011.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Socorro'
+copyright = u'2011, Mozilla'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '2'
+# The full version, including alpha/beta/rc tags.
+release = '2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Socorrodoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'Socorro.tex', u'Socorro Documentation',
+ u'Mozilla', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'socorro', u'Socorro Documentation',
+ [u'Mozilla'], 1)
+]
View
87 docs/crashmover.rst
@@ -0,0 +1,87 @@
+.. index:: crashmover
+
+.. _crashmover-chapter:
+
+Crash Mover
+===========
+
+The :ref:`collector-chapter` dumps all the crashes that it receives into the
+local file system. This application is responsible for transferring
+those crashes into hbase.
+
+**Configuration**::
+
+ import stat
+ import socorro.lib.ConfigurationManager as cm
+
+ #-------------------------------------------------------------------------------
+ # general
+
+ numberOfThreads = cm.Option()
+ numberOfThreads.doc = 'the number of threads to use'
+ numberOfThreads.default = 4
+
+ #-------------------------------------------------------------------------------
+ # source storage
+
+ sourceStorageClass = cm.Option()
+ sourceStorageClass.doc = 'the fully qualified name of the source storage class'
+ sourceStorageClass.default = 'socorro.storage.crashstorage.CrashStorageSystemForLocalFS'
+ sourceStorageClass.fromStringConverter = cm.classConverter
+
+ from config.collectorconfig import localFS
+ from config.collectorconfig import localFSDumpDirCount
+ from config.collectorconfig import localFSDumpGID
+ from config.collectorconfig import localFSDumpPermissions
+ from config.collectorconfig import localFSDirPermissions
+ from config.collectorconfig import fallbackFS
+ from config.collectorconfig import fallbackDumpDirCount
+ from config.collectorconfig import fallbackDumpGID
+ from config.collectorconfig import fallbackDumpPermissions
+ from config.collectorconfig import fallbackDirPermissions
+
+ from config.commonconfig import jsonFileSuffix
+ from config.commonconfig import dumpFileSuffix
+
+ #-------------------------------------------------------------------------------
+ # destination storage
+
+ destinationStorageClass = cm.Option()
+ destinationStorageClass.doc = 'the fully qualified name of the source storage class'
+ destinationStorageClass.default = 'socorro.storage.crashstorage.CrashStorageSystemForHBase'
+ destinationStorageClass.fromStringConverter = cm.classConverter
+
+ from config.commonconfig import hbaseHost
+ from config.commonconfig import hbasePort
+ from config.commonconfig import hbaseTimeout
+
+ #-------------------------------------------------------------------------------
+ # logging
+
+ syslogHost = cm.Option()
+ syslogHost.doc = 'syslog hostname'
+ syslogHost.default = 'localhost'
+
+ syslogPort = cm.Option()
+ syslogPort.doc = 'syslog port'
+ syslogPort.default = 514
+
+ syslogFacilityString = cm.Option()
+ syslogFacilityString.doc = 'syslog facility string ("user", "local0", etc)'
+ syslogFacilityString.default = 'user'
+
+ syslogLineFormatString = cm.Option()
+ syslogLineFormatString.doc = 'python logging system format for syslog entries'
+ syslogLineFormatString.default = 'Socorro Storage Mover (pid %(process)d): %(asctime)s %(levelname)s - %(threadName)s - %(message)s'
+
+ syslogErrorLoggingLevel = cm.Option()
+ syslogErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ syslogErrorLoggingLevel.default = 10
+
+ stderrLineFormatString = cm.Option()
+ stderrLineFormatString.doc = 'python logging system format for logging to stderr'
+ stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(threadName)s - %(message)s'
+
+ stderrErrorLoggingLevel = cm.Option()
+ stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ stderrErrorLoggingLevel.default = 10
View
62 docs/crashreprofilteringreport.rst
@@ -0,0 +1,62 @@
+.. index:: crashreprofilteringreport
+
+.. _crashreprofilteringreport-chapter:
+
+
+Crash Repro Filtering Report
+============================
+
+Introduction
+------------
+
+This page describes a report that assists in analyzing crash data for
+a stack signature in order to try and reproduce a crash and develop a
+reproducible test case.
+
+Details
+-------
+
+for each release pull a data set of one weeks worth of data ranked by
+signature like:
+
+http://crash-stats.mozilla.com/query/query?do_query=1&product=Firefox&version=Firefox%3A3.0.10&date=&range_value=7&range_unit=days&query_search=signature&query_type=contains&query=
+
+the provide a list like this with several fields of interest for
+examing the data
+
+Date Product Version Build OS CPU Reason Address Uptime Comments
+
+but also need to add urls into the version of this report that is
+behind auth. "reason" is not so helpful to me at this stage, but
+others can weigh in on the idea of removing it.
+
+maybe just make it include all these or allow users to pick the fields
+it shows like bugzilla does?
+
+Signature,Crash Address,UUIDProduct,Version,Build,OS,Time,Uptime,Last
+Crash,URL,User Comments
+
+anyway, get something close to what we have now in "Crash Reports in
+PR_MD_SEND"
+
+http://crash-stats.mozilla.com/report/list?product=Firefox&version=Firefox%3A3.0.10&query_search=signature&query_type=contains&query=&date=&range_value=7&range_unit=days&do_query=1&signature=_PR_MD_SEND
+
+next allow the report user to apply filters to build more precise
+queries from the set of reports.. filters might be from any of the
+fields or it would really cool if we could also filter from other
+items in the crash report like the full stack trace and/or module list::
+
+ filter uptime? < 60 seconds
+ and filter address? exactly_matches 0x187d000
+ and fliter url? contains mail.google.com
+ or fliter url? conttains mail.yahoo.com
+ and filter modulelist? does_not_contain "mswsock.dll 5.1.2600.3394"
+
+
+that last example of module list might be a stretch, but would be very
+valuable to check module list for existance or non-existance of binary
+components and their version numbers.
+
+from there we would want to see the results and export to csv to
+import things like url lists into page load testing systems to look
+for reproducible crashers.
View
36 docs/databaseadminfunctions.rst
@@ -0,0 +1,36 @@
+.. index:: database
+
+.. _databaseadminfunctions-chapter:
+
+Database Admin Function Reference
+=================================
+
+What follows is a listing of custom functions written for Socorro in the
+PostgreSQL database which are intended for database administration,
+particularly scheduled tasks. Many of these functions depend on other,
+internal functions which are not documented.
+
+weekly_report_partitions
+------------------------
+
+Purpose: to create new paritions for the reports table and its child
+tables every week.
+
+Called By: weekly cron job
+
+::
+
+ weekly_report_partitions (
+ optional numweeks integer default 2,
+ optional targetdate date default current_date
+ )
+
+ SELECT weekly_report_partitions();
+ SELECT weekly_report_partitions(3,'2011-11-09');
+
+numweeks
+ number of weeks ahead to create partitions
+targetdate
+ date for the starting week, if not today
+
+
View
374 docs/databaseschema.rst
@@ -0,0 +1,374 @@
+.. index:: databaseschema
+
+.. _databaseschema-chapter:
+
+Out-of-Date Data Warning
+========================
+
+While portions of this doc are still relevant and interesting for
+current socorro usage, be aware that it is extremely out of date
+when compared to current schema.
+
+Database Schema
+===============
+
+Introduction
+------------
+
+Socorro is married to the PostgreSQL database: It makes use of a
+significant number of PostrgeSQL and psycopg2 (python) features and
+extensions. Making a database-neutral API has been explored, and for
+now is not being pursued.
+
+The tables can be divided into three major categories: crash data,
+aggregate reporting and process control.
+
+
+crash data
+----------
+
+.. image:: SocorroSchema.CrashData.20090722.png
+
+reports
+-------
+
+This table participates in DatabasePartitioning
+
+Holds a lot of data about each crash report::
+
+ Table "reports"
+ Column | Type | Modifiers | Description
+ ---------------------+-----------------------------+-----------------+-------------
+ id | integer | not null serial | unique id
+ client_crash_date | timestamp with time zone | | as reported by client
+ date_processed | timestamp without time zone | | when entered into jobs table
+ uuid | character varying(50) | not null | unique tag for job
+ product | character varying(30) | | name of product ("Firefox")
+ version | character varying(16) | | version of product("3.0.6")
+ build | character varying(30) | | build of product ("2009041522")
+ signature | character varying(255) | | signature of 'top' frame of crash
+ url | character varying(255) | | associated with crash
+ install_age | integer | | in seconds since installed
+ last_crash | integer | | in seconds since last crash
+ uptime | integer | | in seconds since recent start
+ cpu_name | character varying(100) | | as reported by client ("x86")
+ cpu_info | character varying(100) | | as reported by client ("GenuineIntel family 15 model 4 stepping 1")
+ reason | character varying(255) | | as reported by client
+ address | character varying(20) | | memory address
+ os_name | character varying(100) | | name of os ("Windows NT")
+ os_version | character varying(100) | | version of os ("5.1.2600 Service Pack 3")
+ email | character varying(100) | | -- deprecated
+ build_date | timestamp without time zone | | product build date (column build has same info, different format)
+ user_id | character varying(50) | | -- deprecated
+ started_datetime | timestamp without time zone | | when processor starts processing report
+ completed_datetime | timestamp without time zone | | when processor finishes processing report
+ success | boolean | | whether finish was good
+ truncated | boolean | | whether some dump data was removed
+ processor_notes | text | | error messages during monitor processing of report
+ user_comments | character varying(1024) | | if any, by user
+ app_notes | character varying(1024) | | arbitrary, sent by client (exception detail, etc)
+ distributor | character varying(20) | | future use: "Linux distro"
+ distributor_version | character varying(20) | | future use: "Linux distro version"
+
+ Partitioned Child Table
+ Indexes:
+ "reports_aDate_pkey" PRIMARY KEY, btree (id)
+ "reports_aDate_unique_uuid" UNIQUE, btree (uuid)
+ "reports_aDate_date_processed_key" btree (date_processed)
+ "reports_aDate_product_version_key" btree (product, version)
+ "reports_aDate_signature_date_processed_key" btree (signature, date_processed)
+ "reports_aDate_signature_key" btree (signature)
+ "reports_aDate_url_key" btree (url)
+ "reports_aDate_uuid_key" btree (uuid)
+ Check constraints:
+ "reports_aDate_date_check" CHECK (aDate::timestamp without time zone <= date_processed AND date_processed < aDate+WEEK::timestamp without time zone)
+ Inherits: reports
+
+dumps
+-----
+
+This table is deprecated (dump data is stored in the file system) see
+[[DumpingDumpTables]] for more information.
+
+branches
+--------
+
+This table has been replaced by a view of productdims::
+
+ CREATE VIEW branches AS SELECT product,version,branch FROM productdims;
+
+extensions
+----------
+
+This table participates in [[DatabasePartitioning]].
+
+Holds data about what extensions are associated with a given report::
+
+ Table "extensions"
+ Column | Type | Modifiers | Description
+ ------------------+-----------------------------+-----------+-------------
+ report_id | integer | not null | in child: foreign key reference to child of table 'reports'
+ date_processed | timestamp without time zone | | set to time when the row is inserted
+ extension_key | integer | not null | the name of this extension
+ extension_id | character varying(100) | not null | the id of this extension
+ extension_version | character varying(30) | | the version of this extension
+
+ Partitioned Child Table
+ Indexes:
+ "extensions_aDate_pkey" PRIMARY KEY, btree (report_id)
+ "extensions_aDate_report_id_date_key" btree (report_id, date_processed)
+ Check constraints:
+ "extensions_aDate_date_check" CHECK ('aDate'::timestamp without time zone <= date_processed AND date_processed < 'aDate+WEEK'::timestamp without time zone)
+ Foreign-key constraints:
+ "extensions_aDate_report_id_fkey" FOREIGN KEY (report_id) REFERENCES reports_aDate(id) ON DELETE CASCADE
+ Inherits: extensions
+
+frames
+------
+
+This table participates in [[DatabasePartitioning]]
+
+Holds data about the frames in the dump associated with a particular
+report::
+
+ Table "frames"
+ Column | Type | Modifiers | Description
+ ----------------+-----------------------------+-----------+-------------
+ report_id | integer | not null | in child: foreign key reference to child of table reports
+ date_processed | timestamp without time zone | | set to time when the row is inserted (?)
+ frame_num | integer | not null | ordinal: one row per stack-frame per report, from 0=top
+ signature | character varying(255) | | signature as returned by minidump_stackwalk
+
+ Partitioned Child Table
+ Indexes:
+ "frames_aDate_pkey" PRIMARY KEY, btree (report_id, frame_num)
+ "frames_aDate_report_id_date_key" btree (report_id, date_processed)
+ Check constraints:
+ "frames_aDate_date_check" CHECK ('aDate'::timestamp without time zone <= date_processed AND date_processed < 'aDate+WEEK'::timestamp without time zone)
+ Foreign-key constraints:
+ "frames_aDate_report_id_fkey" FOREIGN KEY (report_id) REFERENCES reports_aDate(id) ON DELETE CASCADE
+ Inherits: frames
+
+
+ Aggregate Reporting
+ ===================
+
+ .. image:: SocorroSchema.Aggregate.20090722.png
+
+productdims
+------------
+
+Dimension table that describes the product, version, gecko version
+('branch') and type of release. Note that the release string is
+completely determined by the version string: A version like 'X.Y.Z' is
+'major'. A version with suffix 'pre' is 'development' and a version
+with 'a' or 'b' (alpha or beta) is 'milestone'. Note: current version
+does not conflate os details (see osdims)::
+
+ Table productdims
+ Column | Type | Modifiers | Description
+ ---------+--------------+-----------+-------------
+ id | integer | (serial) |
+ product | text | not null |
+ version | text | not null |
+ branch | text | not null | gecko version
+ release | release_enum | | 'major', 'milestone', 'development'
+ Indexes:
+ "productdims_pkey1" PRIMARY KEY, btree (id)
+ "productdims_product_version_key" UNIQUE, btree (product, version)
+ "productdims_release_key" btree (release)
+
+osdims
+------
+
+Dimension table that describes an operating system name and version.
+Because there are so many very similar Linux versions, the data saved
+here is simplified which allows many different 'detailed version'
+Linuxen to share the same row in this table.::
+
+ Table osdims
+ Column | Type | Modifiers | Description
+ ------------+------------------------+-----------+-------------
+ id | integer | (serial) |
+ os_name | character varying(100) | |
+ os_version | character varying(100) | |
+ Indexes:
+ "osdims_pkey" PRIMARY KEY, btree (id)
+ "osdims_name_version_key" btree (os_name, os_version)
+
+product_visibility
+------------------
+
+Specifies the date-interval during which a given product
+(productdims_id is the foreign key) is of interest for aggregate
+analysis. MTBF obeys start_date, but calculates its own end date as 60
+days later. Top crash by (url|signature) tables obey both start_date
+and end_date. Column ignore is a boolean, default False, which allows
+a product version to be quickly turned off. Note: Supersedes
+mtbfconfig and tcbyurlconfig. (MTBF is not now in use)::
+
+ Table product_visibility
+ Column | Type | Modifiers | Description
+ ----------------+-----------------------------+---------------+-------------
+ productdims_id | integer | not null |
+ start_date | timestamp without time zone | |
+ end_date | timestamp without time zone | |
+ ignore | boolean | default false |
+ Indexes:
+ "product_visibility_pkey" PRIMARY KEY, btree (productdims_id)
+ "product_visibility_end_date" btree (end_date)
+ "product_visibility_start_date" btree (start_date)
+ Foreign-key constraints:
+ "product_visibility_id_fkey" FOREIGN KEY (productdims_id) REFERENCES productdims(id) ON DELETE CASCADE
+
+time_before_failure
+-------------------
+
+Collects daily summary of average (mean) time before failure for each
+product of interest without regard to specific signature.::
+
+ Table time_before_failure
+ Column | Type | Modifiers | Description
+ --------------------+-----------------------------+------------+-------------
+ id | integer | (serial) |
+ sum_uptime_seconds | double precision | not null |
+ report_count | integer | not null |
+ productdims_id | integer | |
+ osdims_id | integer | |
+ window_end | timestamp without time zone | not null |
+ window_size | interval | not null |
+ Indexes:
+ "time_before_failure_pkey" PRIMARY KEY, btree (id)
+ "time_before_failure_os_id_key" btree (osdims_id)
+ "time_before_failure_product_id_key" btree (productdims_id)
+ "time_before_failure_window_end_window_size_key" btree (window_end, window_size)
+ Foreign-key constraints:
+ "time_before_failure_osdims_id_fkey" FOREIGN KEY (osdims_id) REFERENCES osdims(id) ON DELETE CASCADE
+ "time_before_failure_productdims_id_fkey" FOREIGN KEY (productdims_id) REFERENCES productdims(id) ON DELETE CASCADE
+
+top_crashes_by_signature
+------------------------
+
+The "fact" table that associates signatures with crash statistics::
+
+ Table top_crashes_by_signature
+ Column | Type | Modifiers | Description
+ ----------------+-----------------------------+--------------------+-------------
+ id | integer | (serial) |
+ count | integer | not null default 0 |
+ uptime | real | default 0.0 |
+ signature | text | |
+ productdims_id | integer | |
+ osdims_id | integer | |
+ window_end | timestamp without time zone | not null |
+ window_size | interval | not null |
+ Indexes:
+ "top_crashes_by_signature_pkey" PRIMARY KEY, btree (id)
+ "top_crashes_by_signature_osdims_key" btree (osdims_id)
+ "top_crashes_by_signature_productdims_key" btree (productdims_id)
+ "top_crashes_by_signature_signature_key" btree (signature)
+ "top_crashes_by_signature_window_end_idx" btree (window_end DESC)
+ Foreign-key constraints:
+ "osdims_id_fkey" FOREIGN KEY (osdims_id) REFERENCES osdims(id) ON DELETE CASCADE
+ "productdims_id_fkey" FOREIGN KEY (productdims_id) REFERENCES productdims(id) ON DELETE CASCADE
+
+urldims
+-------
+
+A dimensions table that associates an url and its domain with a
+particular id.
+
+For example, given full url
+http://www.whatever.com/some/path?foo=bar&goo=car
+
+the domain is the host name: www.whatever.com
+
+the url is everything before the query part:
+http://www.whatever.com/some/path::
+
+ Table "urldims"
+ Column | Type | Modifiers | Description
+ --------+------------------------+-----------------+-------------
+ id | integer | not null serial | unique id
+ domain | character varying(255) | not null | the hostname
+ url | character varying(255) | not null | the url up to query
+ Indexes:
+ "urldims_pkey" PRIMARY KEY, btree (id)
+ "urldims_url_domain_key" UNIQUE, btree (url, domain)
+
+top_crashes_by_url
+------------------
+
+The "fact" table that associates urls with crash statistics::
+
+ Table top_crashes_by_url
+ Column | Type | Modifiers | Description
+ ----------------+-----------------------------+-----------+-------------
+ id | integer | (serial) |
+ count | integer | not null |
+ urldims_id | integer | |
+ productdims_id | integer | |
+ osdims_id | integer | |
+ window_end | timestamp without time zone | not null |
+ window_size | interval | not null |
+ Indexes:
+ "top_crashes_by_url_pkey" PRIMARY KEY, btree (id)
+ "top_crashes_by_url_count_key" btree (count)
+ "top_crashes_by_url_osdims_key" btree (osdims_id)
+ "top_crashes_by_url_productdims_key" btree (productdims_id)
+ "top_crashes_by_url_urldims_key" btree (urldims_id)
+ "top_crashes_by_url_window_end_window_size_key" btree (window_end, window_size)
+ Foreign-key constraints:
+ "top_crashes_by_url_osdims_id_fkey" FOREIGN KEY (osdims_id) REFERENCES osdims(id) ON DELETE CASCADE
+ "top_crashes_by_url_productdims_id_fkey" FOREIGN KEY (productdims_id) REFERENCES productdims(id) ON DELETE CASCADE
+ "top_crashes_by_url_urldims_id_fkey" FOREIGN KEY (urldims_id) REFERENCES urldims(id) ON DELETE CASCADE
+
+top_crashes_by_url_signature
+----------------------------
+
+Associates count of each signature with a row in top_crashes_by_url
+table::
+
+ Table top_crashes_by_url_signature
+ Column | Type | Modifiers | Description
+ -----------------------+---------+-----------+-------------
+ top_crashes_by_url_id | integer | not null |
+ signature | text | not null |
+ count | integer | not null |
+ Indexes:
+ "top_crashes_by_url_signature_pkey" PRIMARY KEY, btree (top_crashes_by_url_id, signature)
+ Foreign-key constraints:
+ "top_crashes_by_url_signature_fkey" FOREIGN KEY (top_crashes_by_url_id) REFERENCES top_crashes_by_url(id) ON DELETE CASCADE
+
+topcrashurlfactsreports
+-----------------------
+
+Associates a job uuid with comments and a row in the topcrashurlfacts
+table.::
+
+ Table "topcrashurlfactsreports"
+ Column | Type | Modifiers | Description
+ ---------------------+------------------------+-----------------+-------------
+ id | integer | not null serial | unique id
+ uuid | character varying(50) | not null | job uuid string
+ comments | character varying(500) | | ?programmer provided?
+ topcrashurlfacts_id | integer | | crash statistics for a product,os,url,signature and day
+ Indexes:
+ "topcrashurlfactsreports_pkey" PRIMARY KEY, btree (id)
+ "topcrashurlfactsreports_topcrashurlfacts_id_key" btree (topcrashurlfacts_id)
+ Foreign-key constraints:
+ "topcrashurlfactsreports_topcrashurlfacts_id_fkey" FOREIGN KEY (topcrashurlfacts_id) REFERENCES topcrashurlfacts(id) ON DELETE CASCADE
+
+alexa_topsites
+--------------
+
+Stores a weekly dump of the top 1,000 sites as measured by Alexa (csv)::
+
+ Table "public.alexa_topsites"
+ Column | Type | Modifiers
+ --------------+-----------------------------+------------------------
+ domain | text | not null
+ rank | integer | default 10000
+ last_updated | timestamp without time zone | not null default now()
+ Indexes:
+ "alexa_topsites_pkey" PRIMARY KEY, btree (domain)
View
107 docs/databasesetup.rst
@@ -0,0 +1,107 @@
+.. index:: databasesetup
+
+.. _databasesetup-chapter:
+
+
+Database Setup
+==============
+
+
+This app is under development. For progress information see: `Bugzilla
+454438 <https://bugzilla.mozilla.org/show_bug.cgi?id=454438>`_
+
+This is an application that will set up the PostgreSQL database schema
+for Socorro. It starts with an empty database and creates all the
+tables, indexes, constraints, stored procedures and triggers needed to
+run a Socorro instance.
+
+Before this application can be run, however, there have been set up a
+regular user that will be used for the day to day operations. While it
+is not recommended that the regular user have the full set of super
+user privileges, the regular user must be privileged enough to create
+tables within the database.
+
+Before the application that sets up the database can be run, the
+:ref:`commonconfig-chapter` must be set up. The configuration file for this
+app itself is outlined at the end of this page.
+
+
+Running the setupDatabase app
+-----------------------------
+
+``.../scripts/setupDatabase.py``
+
+Configuring setupDatabase app
+-----------------------------
+
+This application relies on its own configuration file as well as the
+common configuration file :ref:`commonconfig-chapter`.
+
+copy the ``.../scripts/config/setupdatabaseconfig.py.dist`` file to
+``.../scripts/config/setupdatabase.py`` and edit the file to make site
+specific changes.
+
+**logFilePathname**
+
+Monitor can log its actions to a set of automatically rotating log
+files. This is the name and location of the logs.::
+
+ logFilePathname = cm.Option()
+ logFilePathname.doc = 'full pathname for the log file'
+ logFilePathname.default = './monitor.log'
+
+**logFileMaximumSize**
+
+This is the maximum size in bytes allowed for a log file. Once this
+number is achieved, the logs rotate and a new log is started.::
+
+ logFileMaximumSize = cm.Option()
+ logFileMaximumSize.doc = 'maximum size in bytes of the log file'
+ logFileMaximumSize.default = 1000000
+
+**logFileMaximumBackupHistory**
+
+The maximum number of log files to keep.::
+
+ logFileMaximumBackupHistory = cm.Option()
+ logFileMaximumBackupHistory.doc = 'maximum number of log files to keep'
+ logFileMaximumBackupHistory.default = 50
+
+**logFileLineFormatString**
+
+A Python format string that controls the format of individual lines in
+the logs::
+
+ logFileLineFormatString = cm.Option()
+ logFileLineFormatString.doc = 'python logging system format for log file entries'
+ logFileLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
+
+**logFileErrorLoggingLevel**
+
+Logging is done in severity levels - the lower the number, the more
+verbose the logs.::
+
+ logFileErrorLoggingLevel = cm.Option()
+ logFileErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ logFileErrorLoggingLevel.default = 10
+
+**stderrLineFormatString**
+
+In parallel with creating log files, Monitor can log to stderr. This
+is a Python format string that controls the format of individual lines
+sent to stderr.::
+
+ stderrLineFormatString = cm.Option()
+ stderrLineFormatString.doc = 'python logging system format for logging to stderr'
+ stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
+
+**stderrErrorLoggingLevel**
+
+Logging to stderr is done in severity levels independently from the
+log file severity levels - the lower the number, the more verbose the
+output to stderr.::
+
+ stderrErrorLoggingLevel = cm.Option()
+ stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ stderrErrorLoggingLevel.default = 40
+
View
136 docs/databasetablesbysource.rst
@@ -0,0 +1,136 @@
+.. index:: database
+
+.. _databasetablesbysource-chapter:
+
+PostgreSQL Database Tables by Data Source
+=========================================
+
+Last updated: 2011-11-11
+
+This document breaks down the tables in the Socorro PostgreSQL database by where their data comes from, rather than by what the table contains. This is a prerequisite to populating a brand-new socorro database or creating synthetic testing workloads.
+
+Manually Populated Tables
+=========================
+
+The following tables have no code to populate them automatically. Initial population and any updating need to be done by hand. Generally there's no UI, either; use queries.
+
+* daily_crash_codes
+* os_name_matches
+* os_names
+* process_types
+* product_release_channels
+* products
+* release_channel_matches
+* release_channels
+* uptime_levels
+* windows_versions
+
+Tables Receiving External Data
+==============================
+
+These tables actually get inserted into by various external utilities. This is most of our "incoming" data.
+
+bugs
+ list of bugs, populated by bugzilla-scraper
+extensions
+ populated by processors
+plugins_reports
+ populated by processors
+raw_adu
+ populated by daily batch job from metrics
+releases_raw
+ populated by daily FTP-scraper
+reports
+ populated by processors
+
+
+Automatically Populated Reference Tables
+========================================
+
+Lookup lists and dimension tables, populated by cron jobs and/or processors based on the above tables.
+
+* addresses
+* domains
+* flash_versions
+* os_versions
+* plugins
+* product_version_builds
+* product_versions
+* reasons
+* reports_bad
+* signatures
+
+Matviews
+========
+
+Reporting tables, designed to be called directly by the mware/UI/reports. Populated by cron job batch.
+
+* bug_associations
+* daily_crashes
+* daily_hangs
+* os_signature_counts
+* product_adu
+* product_signature_counts
+* reports_clean
+* reports_user_info
+* reports_duplicates
+* signature_bugs_rollup
+* signature_first
+* signature_products
+* signature_products_rollup
+* tcbs
+* uptime_signature_counts
+
+Application Management Tables
+=============================
+
+These tables are used by various parts of the application to do other things than reporting. They are populated/managed by those applications.
+
+* email campaign tables
+
+ * email_campaigns
+ * email_campaigns_contacts
+ * email_contacts
+
+* processor management tables
+
+ * jobs
+ * priorityjobs
+ * priority_jobs_*
+ * processors
+ * server_status
+
+* UI management tables
+
+ * sessions
+
+* monitoring tables
+
+ * replication_test
+
+* cronjob and database management
+
+ * cronjobs
+ * report_partition_info
+
+Depreciated Tables
+==================
+
+These tables are supporting functionality which is scheduled to be removed over the next few versions of Socorro. As such, we are ignoring them.
+
+* alexa_topsites
+* builds
+* frames
+* osdims
+* priorityjobs_log
+* priorityjobs_logging_switch
+* product_visibility
+* productdims
+* productdims_version_sort
+* release_build_type_map
+* signature_build
+* signature_productdims
+* top_crashes_by_signature
+* top_crashes_by_url
+* top_crashes_by_url_signature
+* urldims
View
116 docs/deferredcleanup.rst
@@ -0,0 +1,116 @@
+.. index:: deferredcleanup
+
+.. _deferredcleanup-chapter:
+
+
+Deferred Cleanup
+================
+
+When the :ref:`collector-chapter` throttles the flow of crash dumps, it saves
+deferred crashes into :ref:`deferredjobstorage-chapter`. These JSON/dump pairs will
+live in deferred storage for a configurable number of days. It is the
+task of the deferred cleanup application to implement the policy to
+delete old crash dumps.
+
+The deferred cleanup application is a command line app meant to be run
+via as a cron job. It should be set to run once every twenty-four
+hours.
+
+Configuration
+-------------
+
+deferredcleanup uses the common configuration for to get the constant
+deferredStorageRoot. For setup of common configuration, see
+:ref:`commonconfig-chapter`.
+
+deferredcleanup also has an executable configuration file of its own.
+A sample file is found at
+``.../scripts/config/deferredcleanupconfig.py.dist``. Copy this file to
+``.../scripts/config/deferredcleanupconfig.py`` and edit it for site
+specific settings.
+
+In each case where a site specific value is desired, replace the value
+for the .default member.
+
+**maximumDeferredJobAge**
+
+This constant specifies how many days deferred jobs are allowed to
+stay in deferred storage. Job deletion is permanent.::
+
+ maximumDeferredJobAge = cm.Option()
+ maximumDeferredJobAge.doc = 'the maximum number of days that deferred jobs stick around'
+ maximumDeferredJobAge.default = 2
+
+**dryRun**
+
+Used during testing and development, this prevents deferredcleanup
+from actually deleting things.::
+
+ dryRun = cm.Option()
+ dryRun.doc = "don't really delete anything"
+ dryRun.default = False
+ dryRun.fromStringConverter = cm.booleanConverter
+
+**logFilePathname**
+
+Deferredcleanup can log its actions to a set of automatically rotating
+log files. This is the name and location of the logs.::
+
+ logFilePathname = cm.Option()
+ logFilePathname.doc = 'full pathname for the log file'
+ logFilePathname.default = './processor.log'
+
+**logFileMaximumSize**
+
+This is the maximum size in bytes allowed for a log file. Once this
+number is achieved, the logs rotate and a new log is started.::
+
+ logFileMaximumSize = cm.Option()
+ logFileMaximumSize.doc = 'maximum size in bytes of the log file'
+ logFileMaximumSize.default = 1000000
+
+**logFileMaximumBackupHistory**
+
+The maximum number of log files to keep.::
+
+ logFileMaximumBackupHistory = cm.Option()
+ logFileMaximumBackupHistory.doc = 'maximum number of log files to keep'
+ logFileMaximumBackupHistory.default = 50
+
+**logFileLineFormatString**
+
+A Python format string that controls the format of individual lines in
+the logs::
+
+ logFileLineFormatString = cm.Option()
+ logFileLineFormatString.doc = 'python logging system format for log file entries'
+ logFileLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
+
+**logFileErrorLoggingLevel**
+
+Logging is done in severity levels - the lower the number, the more
+verbose the logs.::
+
+ logFileErrorLoggingLevel = cm.Option()
+ logFileErrorLoggingLevel.doc = 'logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ logFileErrorLoggingLevel.default = 20
+
+**stderrLineFormatString**
+
+In parallel with creating log files, Monitor can log to stderr. This
+is a Python format string that controls the format of individual lines
+sent to stderr.::
+
+ stderrLineFormatString = cm.Option()
+ stderrLineFormatString.doc = 'python logging system format for logging to stderr'
+ stderrLineFormatString.default = '%(asctime)s %(levelname)s - %(message)s'
+
+**stderrErrorLoggingLevel**
+
+Logging to stderr is done in severity levels independently from the
+log file severity levels - the lower the number, the more verbose the
+output to stderr.::
+
+ stderrErrorLoggingLevel = cm.Option()
+ stderrErrorLoggingLevel.doc = 'logging level for the logging to stderr (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)'
+ stderrErrorLoggingLevel.default = 40
View
22 docs/deferredjobstorage.rst
@@ -0,0 +1,22 @@
+.. index:: deferredjobstorage
+
+.. _deferredjobstorage-chapter:
+
+
+Deferred Job Storage
+====================
+
+Deferred storage is where the JSON/dump pairs are saved if they've
+been filtered out by :ref:`collector-chapter` throttling. The location of the
+deferred job storage is determined by the configuration parameter
+deferredStorageRoot found in the :ref:`commonconfig-chapter`.
+
+JSON/dump pairs that are saved in deferred storage are not likely to
+ever be processed further. They are held for a configurable number of
+days until deleted by :ref:`deferredcleanup-chapter`.
+
+Occasionally, a developer will request a report via :ref:`reporter-chapter` on
+a job that was saved in deferred storage. :ref:`monitor-chapter` will look for
+the job in deferred storage if it cannot find it in standard storage.
+
+For more information on the storage technique, see :ref:`filesystem-chapter`
View
25 docs/deployment.rst
@@ -0,0 +1,25 @@
+.. index:: deployment
+
+.. _deployment-chapter:
+
+Deployment
+==========
+
+Introduction
+------------
+
+Below are general deployment instructions for installations of Socorro.
+
+Outage Page
+-----------
+if the system is to be taken down for maintenance, these steps will
+show users an outage page during the maintenance period
+
+* backup webapp-php/index.php
+* You can copy webapp-php/docs/outage.php over
+ webapp-php/index.php and all traffic will be served this outage
+ message.
+* Do work
+* copy backup over webapp-php/index.php
+
+**add other task instructions here**
View
27 docs/development.rst
@@ -0,0 +1,27 @@
+.. index:: development
+
+.. _development-chapter:
+
+Development Discussions
+=======================
+
+.. toctree::
+ :maxdepth: 2
+
+ codingconventions
+ glossary
+ standalone
+ unittesting
+ crashreprofilteringreport
+ databasepartitioning
+ diskperformancetests
+ dumpingdumptables
+ jsondumpstorage
+ processeddumpstorage
+ reportdatabasedesign
+ codeanddatabaseupdate
+ databaseschema
+ package
+ schema
+ databasesetup
+ commonconfig
View
183 docs/diskperformancetests.rst
@@ -0,0 +1,183 @@
+.. index:: diskperformancetests
+
+.. _diskperformancetests-chapter:
+
+
+Disk Performance Tests
+======================
+
+Introduction
+------------
+
+Any DBMS for a database which is larger than memory can be no faster
+than disk speed. This document outlines a series of tests for testing
+disk speed to determine if you have an issue. Written originally by
+PostgreSQL Experts Inc. for Mozilla.
+
+Running Tests
+-------------
+
+Note: all of the below require you to have plenty of disk space
+available. And their figures are only reliable if nothing else is
+running on the system.
+
+
+**Simplest Test: The DD Test**
+
+This test measures the most basic single-threaded disk access: a large
+sequential write, followed by a large sequential read. It is relevant
+to database performance because it gives you a maximum speed for
+sequential scans for large tables. Real table scans are generally
+about 30% of this maximum.
+
+dd is a Unix command line utility which simply writes to a block
+device. We use it for this 3-step test. The other thing you need to
+know for this test is your RAM size.
+
+1. We create a large file which is 2x the size of RAM, and synch it to
+ disk. This makes sure that we get the real sustained write rate,
+ because caching can have little effect. Since there are 125000 blocks
+ per GB (8k blocksize is used because it's what Postgres uses), if we
+ had 8GB of RAM, we would run the following::
+
+ time sh -c "dd if=/dev/zero of=ddfile bs=8k count=1000000 && sync"
+
+ dd will report a time and write rate to us, and "time" will report
+ a larger time. The time and rate reported by dd represents the rate
+ without any lag or synch time; divide the data size by the time
+ reported by "time" for synchronous file writing rate.
+
+2. Next we want to write another large file, this one the size of RAM,
+ in order to flush out the FS cache so that we can read directly
+ from disk later.::
+
+ dd if=/dec/zero of=ddfile2 bs=8K count=500000
+
+3. Now, we want to read the first file back. Since the FS cache is
+ full from the second file, this should be 100% disk access::
+
+ time dd if=ddfile of=/dev/null bs=8k
+
+This time, "time" and dd will be very close together; any difference
+will be strictly storage lag time.
+
+
+Bonnie++
+--------
+
+Bonnie++ is a more sophisticated set of tests which tests random reads
+and writes, as well as seeks, and file creation and deletion
+operations. For a modern system, you want to use the last version,
+1.95, downloaded from http://www.coker.com.au/bonnie++/experimental/
+This final version of bonnie++ supports concurrency and measures lag
+time. However, it is not available in package form in most OSes, so
+you'll have to compile it using g++.
+
+Again, for Mozilla we want to test performance for a database which is
+larger than RAM, since that's what we have. Therefore, we're going to
+run a concurrent Bonnie++ test where the total size of the files is
+about 150% of RAM, forcing the use of disk. We're also going to run 8
+threads to simulate concurrent file access. Our command line for a
+machine with 16GB RAM is::
+
+ bonnie++ -d /path/to/storage -c 8 -r 16000 -n 100
+
+The results we get back look something like this::
+
+ Version 1.95 ------Sequential Output------ --Sequential Input- --Random-
+ Concurrency 8 -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
+ Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
+ tm-breakpad0 32000M 757 99 71323 16 30594 5 2192 99 57555 4 262.5 13
+ Latency 15462us 6918ms 4933ms 11096us 706ms 241ms
+ Version 1.95 ------Sequential Create------ --------Random Create--------
+ tm-breakpad01-maste -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
+ files /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP /sec %CP
+ 100 44410 75 +++++ +++ 72407 81 45787 77 +++++ +++ 63167 72
+ Latency 9957us 477us 533us 649us 93us 552us
+
+So, the interesting parts of this are:
+
+Sequential Output: Block: this is sequential writes like dd does. It's
+70MB/s.
+
+Sequential Input: Block: this is sequential reads from disk. It's
+57MB/s.
+
+Sequential Output: Rewrite: is reading, then writing, a file which has
+been flushed to disk. This rate will be lower than either of the
+above, and is at 30MB/s.
+
+Random: Seeks: this is how many individual blocks Bonnie can seek to
+per second; it's a fast 262.
+
+Latency: this is the full round-trip lag time for the mentioned
+operation. On this platform, these times are catastrophically bad; 1/4
+second round-trip to return a single random block, and 3/4 seconds to
+return the start of a large file.
+
+The figures on file creations and deletion are generally less
+interesting to databases. The +++++ are for runs that were so fast the
+error margin makes the figures meaningless; for better figures,
+increase -n.
+
+IOZone
+------
+
+Now, if you don't think Bonnie++ told you enough, you'll want to run
+Iozone. Iozone is a benchmark mostly know for creating pretty graphs
+(http://www.iozone.org/) of filesystem performance with different
+file, batch, and block sizes. However, this kind of comprehensize
+profiling is completely unnecessary for a DBMS, where we already know
+the file access pattern, and can take up to 4 days to run. So do not
+run Iozone in automated (-a) mode!
+
+Instead, run a limited test. This test will still take several hours
+to run, but will return a more limited set of relevant results. Run
+this on a 16GB system with 8 cores, from a directory on the storage
+you want to measure::
+
+ iozone -R -i 0 -i 1 -i 2 -i 3 -i 4 -i 5 -i 8 -l 6 -u 6 -r 8k -s 4G -F f1 f2 f3 f4 f5 f6
+
+This runs the following tests: write/read, rewrite/reread,
+random-read/write, read-backwards, re-write-record, stride-read,
+random mix. It does these tests using 6 concurrent processes, a block
+size of 8k (Postgres' block size) for 4G files named f1 to f6. The
+aggregate size of the files is 24G, so that they won't all fit in
+memory at once.
+
+In theory, the relevance of these tests to database activity is the
+following:
+
+write/read: basic sequential writes and reads.
+
+rewrite/reread: writes and reads of frequently accessed tables (in memory)
+
+random-read/write: index access, and writes of individual rows
+
+read-backwards: might be relevant to reverse index scans.
+
+re-write-record: frequently updated row behavior
+
+stride-read: might be relevant to bitmapscan
+
+random mix: general database access average behavior.
+
+The results you get will look like this::
+
+ Children see throughput for 6 initial writers = 108042.81 KB/sec
+ Parent sees throughput for 6 initial writers = 31770.90 KB/sec
+ Min throughput per process = 13815.83 KB/sec
+ Max throughput per process = 35004.07 KB/sec
+ Avg throughput per process = 18007.13 KB/sec
+ Min xfer = 1655408.00 KB
+
+And so on through all the tests. These results are pretty
+self-explanatory, except that I have no idea what the difference
+between "Children see" and "Parent sees" means. Iozone documentation
+is next-to-nonexistant.
+
+Note: IOZone appears to have several bugs, and places where its
+documentation and actual features don't match. Particularly, it
+appears to have locking issues in concurrent access mode for some
+writing activity so that concurrency throughput may be lower than
+actual.
View
89 docs/dumpingdumptables.rst
@@ -0,0 +1,89 @@
+.. index:: dumpingdumptables
+
+.. _dumpingdumptables-chapter:
+
+
+Dumping Dump Tables
+===================
+
+A work item that came out of the Socorro Postgres work week is to dump the dump tables and store cooked dumps as gzipped files.
+Drop dumps table
+
+convert each dumps table row to a compressed file on disk
+
+Bugzilla
+--------
+
+https://bugzilla.mozilla.org/show_bug.cgi?id=484032
+
+Library support
+---------------
+
+'done' as of 2009-05-07 in socorro.lib.dmpStorage (Coding, testing is done; integration testing is done, 'go live' is today)
+Socorro UI
+
+/report/index/{uuid}
+
+* Will stop using the dumps table.
+* Will start using gzipped files
+ * Will use the report uuid to locate the dump on a file system
+ * Will use apache mod-rewrite to serve the actual file. The rewrite
+ rule is based on the uuid, and is 'simple':
+ AABBCCDDEEFFGGHHIIJJKKLLM2090308.jsonz => AA/BB/AABBCCDDEEFFGGHHIIJJKKLLM2090308.jsonz
+ * report/index will include a link to JSON dump
+
+ link rel='alternate' type='application/json' href='/reporter/dumps/cdaa07ae-475b-11dd-8dfa-001cc45a2ce4.jsonz'
+
+Dump file format
+----------------
+
+* Will be gzip compressed JSON encoded cooked dump files
+* Partial JSON file
+* Full JSONZ file
+
+On Disk Location
+----------------
+
+ application.conf dumpPath Example for kahn $config'dumpPath'? = '/mnt/socorro_dumps/named';
+
+In the dumps directory we will have an .htaccess file::
+
+ AddType "application/json; charset=UTF-8" jsonz
+ AddEncoding gzip jsonz
+
+Webhead will serve these files as::
+
+ Content-Type: application/json; charset=utf-8
+ Content-Encoding: gzip
+
+**Note:* You'd expect the dump files to be named json.gz, but this is
+broken in Safari. By setting HTTP headers and naming the file jsonz,
+an unknown file extension, this works across browsers.
+
+Socorro UI
+----------
+
+* Existing URL won't change.
+* Second JSON request back to server will load jsonz file
+
+Example:
+
+* http://crash-stats.mozilla.com/report/index/d92ebf79-9858-450d-9868-0fe042090211
+* http://crash-stats.mozilla.com/dump/d92ebf79-9858-450d-9868-0fe042090211.jsonz
+
+mod rewrite rules will match /dump/.jsonz and change them to access a file share.
+
+Future Enhancement
+------------------
+
+A future enhancement if we find webheads are high CPU would be to move
+populating the report/index page to client side.
+
+Test Page
+---------
+
+http://people.mozilla.org/~aking/Socorro/dumpingDump/json-test.html -
+Uses browser to decompress a gzip compressed JSON file during an AJAX
+request, pulls it apart and appends to the page.
+
+Test file made with gzip dump.json
View
12 docs/filesystem.rst
@@ -0,0 +1,12 @@
+.. index:: filesystem
+
+.. _filesystem-chapter:
+
+
+File System
+===========
+
+Socorro uses two similar file system storage schemes in two distinct
+places within the system. Raw crash dumps from the field use a system
+called :ref:`jsondumpstorage-chapter` while at the other end, processed dumps use the
+:ref:`processeddumpstorage-chapter` scheme.
View
112 docs/glossary.rst
@@ -0,0 +1,112 @@
+.. index:: glossary
+
+.. _glossary-chapter:
+
+
+Glossary
+========
+
+**Build**: a date encoding used to identify when a client was compiled.
+(submission metadata)
+
+**Crash Report Details Page** - A crash stats page displaying all known
+details of a crash
+
+**Crash Dump/Metadata pair** - shorthand for The pair of Raw Crash Dump
+and corresponding Raw Crash Metadata
+
+:ref:`deferredjobstorage-chapter`: a file system location where Crash Dump/Metadata
+pair are kept without being processed.
+
+**Dump File**: See Raw Crash Dump, don't use this term it makes me giggle
+
+**Job**: a job queue item for a Raw Crash Dump that needs to be processed
+
+:ref:`jsondumpstorage-chapter`: the Python module that implements
+:ref:`filesystem-chapter`
+
+**Materialized view**: the tables in the database containing the data for
+used in statistical analysis. Including: [[MeanTimeBeforeFailure]],
+:ref:`topcrashersbysignature-chapter`, :ref:`topcrashersbyurl-chapter`. The "Trend Reports" from the
+:ref:`ui-chapter` display information from these tables.
+
+**Minidump**: see 'raw crash dump'
+
+**Minidump_stackwalk**: an application from the Breakpad project that
+takes a raw dump file, marries it with symbols and produces output
+usable by developers. This application is invoked by
+:ref:`processor-chapter`.
+
+**Monitor**: the Socorro application in charge of queuing jobs. See
+:ref:`monitor-chapter`
+
+**OOID**: A crash report ID. Originally a 32bit value, the original legacy
+system stored it in the database as a hexidecimal text form. Each
+crash is assigned an OOID by the :ref:`collector-chapter` when the crash is
+recieved.
+
+**Platform**: the OS that a client runs on. This term has been
+historically a point of confusion and it is preferred that the term OS
+or Client OS be used instead.
+
+:ref:`processeddumpstorage-chapter`: the disk location where the output files of the
+minidump_stackwalk program are stored. The actual files are stored
+with a .jsonz extension.
+
+**Processor**: the Socorro application in charge of applying
+minidump_stackwalk to queued jobs. See :ref:`processor-chapter`
+
+**Raw Crash Dump, Raw Dump**: the data sent from a client to Socorro
+containing the state of the application at the time of failure. It is
+paired with a Raw Crash Metadata file.
+
+**Raw Crash Metadata** - the metadata sent from a client to Socorro to
+describe the Raw Crash. It is saved in JSON format, not to be confused
+with a Cooked Crash Dump.
+
+**Raw JSON file**: See Crash Dump Metadata... a file in the JSON format
+containing metadata about a 'dump file'. Saved with a '.json' suffix.
+
+**Release**: a categorization of an application's product name and
+version. The categories are: "major", "milestone", or "development".
+Within the database, an enum called ReleaseEnum? represents these
+categories.
+
+**Reporter**: another name for the :ref:`ui-chapter`
+
+**Skip List**: lists of signature regular expressions used in generating a
+crash's overall signature in the :ref:`processor-chapter`. see
+:ref:`signaturegeneration-chapter`
+
+:ref:`standardjobstorage-chapter`: a file system location where JSON/dump pairs are
+kept for processing
+
+**Throttling**: statistically, we don't have to save every single crash.
+This option of the :ref:`collector-chapter` configuration allows us to
+selectively throw away dumps.
+
+**Trend Reports**: the pages in the :ref:`ui-chapter` that display the data from
+the materialized views.
+
+**UUID**: a univeral unique identifier. Term is being deprecated in favor
+of OOID.
+
+**Web head**: a machine that runs :ref:`collector-chapter`
+
+
+.. toctree::
+ :maxdepth: 2
+
+ deferredjobstorage
+ jsondumpstorage
+ processeddumpstorage
+ standardjobstorage
+ topcrashersbyurl
+ topcrashersbysignature
+ signaturegeneration
+ crashmover
+ collector
+ reporter
+ monitor
+ filesystem
+ deferredcleanup
View
45 docs/index.rst
@@ -0,0 +1,45 @@
+.. Socorro documentation master file, created by
+ sphinx-quickstart on Wed Sep 21 14:59:08 2011.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root ``toctree`` directive.
+
+Socorro
+=======