Merge pull request #1 from it-dot-com/issue/1

Fixes Issue #1 - adds support for PostLinks and Comments

Author: musically-ut

README.md

@@ -24,6 +24,7 @@ Schema hints are taken from [a post on Meta.StackExchange](http://meta.stackexch
    - `python load_into_pg.py Tags` (not present in earliest dumps)
    - `python load_into_pg.py Users`
    - `python load_into_pg.py Votes`
+   - `python load_into_pg.py PostLinks`
    - `python load_into_pg.py PostHistory`
    - `python load_into_pg.py Comments`
  - Finally, after all the initial tables have been created:
@@ -39,7 +40,6 @@ Schema hints are taken from [a post on Meta.StackExchange](http://meta.stackexch
  - It prepares some indexes and views which may not be necessary for your analysis.
  - The `Body` field in `Posts` table is NOT populated by default. You have to use `--with-post-body` argument to include it.
  - The `EmailHash` field in `Users` table is NOT populated.
- - Some tables (e.g. `PostHistory` and `Comments`) are missing.
 
 ### Sept 2011 data dump
 

load_into_pg.py

@@ -119,7 +119,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
 parser = argparse.ArgumentParser()
 parser.add_argument( 'table'
                    , help    = 'The table to work on.'
-                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes', 'PostHistory', 'Comments']
+                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes', 'PostLinks', 'PostHistory', 'Comments']
                    )
 
 parser.add_argument( '-d', '--dbname'
@@ -187,6 +187,23 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
       , 'Name'
       , 'Date'
     ]
+elif table == 'PostLinks':
+    keys = [
+        'Id'
+      , 'CreationDate'
+      , 'PostId'
+      , 'RelatedPostId'
+      , 'LinkTypeId'
+    ]
+elif table == 'Comments':
+    keys = [
+        'Id'
+      , 'PostId'
+      , 'Score'
+      , 'Text'
+      , 'CreationDate'
+      , 'UserId'
+    ]
 elif table == 'Votes':
     keys = [
         'Id'

sql/Comments_post.sql

@@ -1,5 +1,5 @@
 -- hash index takes too long to create
-CREATE INDEX cmnts_post_type_id_idx ON Comments USING btree (Score)
+CREATE INDEX cmnts_score_idx ON Comments USING btree (Score)
        WITH (FILLFACTOR = 100);
 CREATE INDEX cmnts_postid_idx ON Comments USING hash (PostId)
        WITH (FILLFACTOR = 100);
@@ -8,4 +8,4 @@ CREATE INDEX cmnts_revguid_idx ON Comments USING btree (RevisionGUID)
 CREATE INDEX cmnts_creation_date_idx ON Comments USING btree (CreationDate)
        WITH (FILLFACTOR = 100);
 CREATE INDEX cmnts_userid_idx ON Comments USING btree (UserId)
-       WITH (FILLFACTOR = 100);
+       WITH (FILLFACTOR = 100);

sql/Comments_pre.sql

@@ -1,9 +1,9 @@
 DROP TABLE IF EXISTS Comments CASCADE;
 CREATE TABLE Comments (
-    Id            int  PRIMARY KEY   ,
-    PostId        int                ,
-    Score         int                ,
-    Post_Text     text               ,
-    CreationDate  timestamp not NULL ,
-    UserId        int
+    Id                     int PRIMARY KEY    ,
+    PostId                 int not NULL       , 
+    Score                  int not NULL       ,
+    Text                   text               ,
+    CreationDate           timestamp not NULL , 
+    UserId                 int
 );

sql/PostLinks_post.sql

@@ -0,0 +1,5 @@
+-- The hash index is too slow to create
+CREATE INDEX postlinks_post_id_idx on PostLinks USING btree (PostId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX postlinks_related_post_id_idx on PostLinks USING btree (RelatedPostId)
+       WITH (FILLFACTOR = 100);

sql/PostLinks_pre.sql

@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS PostLinks CASCADE;
+CREATE TABLE PostLinks (
+   Id                int         PRIMARY KEY ,
+   CreationDate      timestamp   not NUll    ,
+   PostId            int         not NULL    ,
+   RelatedPostId     int         not NULL    ,
+   LinkTypeId        int         not Null
+);