Permalink
Browse files

Add sample data, README, test scripts

  • Loading branch information...
1 parent 45b8bcc commit 7fd593da4987b2c74a98ea5adf523658a7370648 @jwilder committed Jan 26, 2012
Showing with 183 additions and 1 deletion.
  1. +3 −1 .gitignore
  2. +103 −0 README.md
  3. +2 −0 collection-stats.py
  4. 0 examples/__init__.py
  5. +30 −0 examples/models.py
  6. +30 −0 examples/testdata.py
  7. +2 −0 index-stats.py
  8. +2 −0 redundant-indexes.py
  9. +1 −0 requirements.txt
  10. +9 −0 run-mongo.sh
  11. +1 −0 setup.sh
View
@@ -1,2 +1,4 @@
-virtualenv
+.data
+*.pyc
distribute-0.6.10.tar.gz
+virtualenv
View
103 README.md
@@ -0,0 +1,103 @@
+### mongodb-tools - Utilities for working MongoDB
+
+* collection-stats.py - Display statistics about the collections in all databases
+* index-stats.py - Displays statistics about the indexes in all databases
+* redudant-indexes.py - Finds indexes that may redundant
+
+## Installation
+
+ git clone https://github.com/jwilder/mongodb-tools
+ cd mongodb-tools
+ ./setup.sh
+ source virtualenv/bin/activate
+
+## Test D
+
+ In on terminal run:
+
+ $ ./run-mongo.sh
+
+
+ In another terminal run:
+
+ $ python examples/testdata.py
+
+
+## collection-stats.py ##
+
+ $ ./collection-stats.py
+
+ Checking DB: examples2.system.indexes
+ Checking DB: examples2.things
+ Checking DB: examples1.system.indexes
+ Checking DB: examples1.address
+ Checking DB: examples1.typeless_address
+ Checking DB: examples1.user
+ Checking DB: examples1.typeless_user
+
+
+ +----------------------------+--------+--------+---------+--------------+---------+------------+
+ | Collection | Count | % Size | DB Size | Avg Obj Size | Indexes | Index Size |
+ +----------------------------+--------+--------+---------+--------------+---------+------------+
+ | examples1.address | 2 | 0.0% | 184.00b | 92.00b | 2 | 15.97K |
+ | examples1.system.indexes | 9 | 0.0% | 912.00b | 101.33b | 0 | 0.00b |
+ | examples1.typeless_address | 2 | 0.0% | 216.00b | 108.00b | 1 | 7.98K |
+ | examples2.system.indexes | 2 | 0.0% | 164.00b | 82.00b | 0 | 0.00b |
+ | examples1.typeless_user | 101879 | 26.7% | 10.10M | 104.00b | 3 | 8.18M |
+ | examples1.user | 101879 | 36.0% | 13.60M | 140.00b | 3 | 15.20M |
+ | examples2.things | 100000 | 37.3% | 14.11M | 148.00b | 2 | 5.67M |
+ +----------------------------+--------+--------+---------+--------------+---------+------------+
+ Total Documents: 303773
+ Total Data Size: 37.82M
+ Total Index Size: 29.08M
+ RAM Headroom: 2.87G
+ RAM Used: 2.74G (61.6%)
+ Available RAM Headroom: 1.10G
+
+## index-stats.py
+
+ $ ./index-stats.py
+
+
+ Checking DB: examples2.system.indexes
+ Checking DB: examples2.things
+ Checking DB: examples1.system.indexes
+ Checking DB: examples1.address
+ Checking DB: examples1.typeless_address
+ Checking DB: examples1.user
+ Checking DB: examples1.typeless_user
+
+ Index Overview
+ +----------------------------+------------------------+--------+------------+
+ | Collection | Index | % Size | Index Size |
+ +----------------------------+------------------------+--------+------------+
+ | examples1.address | _id_ | 0.0% | 7.98K |
+ | examples1.address | _types_1 | 0.0% | 7.98K |
+ | examples1.typeless_address | _id_ | 0.0% | 7.98K |
+ | examples1.typeless_user | _id_ | 10.9% | 3.17M |
+ | examples1.typeless_user | address_id_1 | 10.9% | 3.17M |
+ | examples1.typeless_user | typeless_address_ref_1 | 6.4% | 1.85M |
+ | examples1.user | _id_ | 10.9% | 3.17M |
+ | examples1.user | _types_1_address_id_1 | 13.2% | 3.84M |
+ | examples1.user | _types_1_address_ref_1 | 28.2% | 8.20M |
+ | examples2.things | _id_ | 10.7% | 3.11M |
+ | examples2.things | _types_1 | 8.8% | 2.56M |
+ +----------------------------+------------------------+--------+------------+
+
+ Top 5 Largest Indexes
+ +-------------------------+------------------------+--------+------------+
+ | Collection | Index | % Size | Index Size |
+ +-------------------------+------------------------+--------+------------+
+ | examples1.user | _types_1_address_ref_1 | 28.2% | 8.20M |
+ | examples1.user | _types_1_address_id_1 | 13.2% | 3.84M |
+ | examples1.typeless_user | _id_ | 10.9% | 3.17M |
+ | examples2.things | _id_ | 10.7% | 3.11M |
+ | examples2.things | _types_1 | 8.8% | 2.56M |
+ +-------------------------+------------------------+--------+------------+
+
+ Total Documents: 303773
+ Total Data Size: 37.82M
+ Total Index Size: 29.08M
+ RAM Headroom: 2.87G
+ RAM Used: 2.73G (61.4%)
+ Available RAM Headroom: 1.11G
View
2 collection-stats.py 100644 → 100755
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
"""
This script prints some basic collection stats about the size of the
collections and their indexes.
View
No changes.
View
@@ -0,0 +1,30 @@
+from mongoengine import *
+
+class Address(Document):
+ street = StringField()
+
+class TypelessAddress(Document):
+ meta = {"index_types" : False}
+ street = StringField()
+
+class User(Document):
+ meta = {
+ "indexes": [("address_ref"),
+ ("address_id")
+ ]}
+
+ address_ref = ReferenceField("Address")
+ address_id = ObjectIdField()
+
+class TypelessUser(Document):
+ meta = {
+ "indexes": [("address_id"),
+ ("typeless_address_ref")
+ ],
+ "index_types" : False}
+ address_id = ObjectIdField()
+ typeless_address_ref = ReferenceField("TypelessAddress")
+
+
+class Things(Document):
+ long_field = StringField()
View
@@ -0,0 +1,30 @@
+from bson.objectid import ObjectId
+from examples.models import User, Address, Things, TypelessAddress, TypelessUser
+from mongoengine.connection import connect
+
+def add_dataset1():
+ address = Address(street="123 Main St")
+ address.save()
+ address.reload()
+ typeless_address = TypelessAddress(street="123 Main St")
+ typeless_address.save()
+ typeless_address.reload()
+
+ for i in range(0, 100000):
+ user1 = User(address_ref=address,address_id=address.id)
+ user1.save(safe=False)
+ user2 = TypelessUser(address_id=address.id,
+ typeless_address=typeless_address)
+ user2.save(safe=False)
+
+connect('examples1')
+add_dataset1()
+
+def add_dataset2():
+
+ for i in range(0, 100000):
+ thing = Things(long_field="http://www.somelongurl.com?foo=bar&id=%s" % ObjectId())
+ thing.save(safe=False)
+
+connect('examples2')
+add_dataset2()
View
2 index-stats.py 100644 → 100755
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
"""
This script prints some basic collection stats about the size of the
collections and their indexes.
View
2 redundant-indexes.py 100644 → 100755
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
"""
This is a simple script to print out potentially redundant indexes in a mongdb instance.
For example, if an index is defined on {field1:1,field2:1} and there is another index
View
@@ -1,3 +1,4 @@
pymongo==2.1
PrettyTable
psutil==0.3.0
+mongoengine==0.5.0
View
@@ -0,0 +1,9 @@
+#!/bin/bash
+MONGODB_INSTALLED=`which mongod`
+if [ "MONGODB_INSTALLED" == "" ]; then
+ echo -e "mongodb NOT found."
+ exit 1
+fi
+
+mkdir -p ./.data
+mongod --dbpath ./.data
View
@@ -10,5 +10,6 @@ virtualenv --no-site-packages virtualenv
source virtualenv/bin/activate
pip install -r requirements.txt
+export PYTHONPATH=.:$PYTHONPATH
echo "Run \"source virtualenv/bin/activate\" to activate this virtualenv."

0 comments on commit 7fd593d

Please sign in to comment.