Refactored and added support for splitting up tags.

SamSaffron · Jul 8, 2009 · 9606388 · 9606388
1 parent ac8cfc6
commit 9606388
Show file tree

Hide file tree

Showing 9 changed files with 5,802 additions and 196 deletions.
diff --git a/DumpReader.cs b/DumpReader.cs
@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Data;
+using System.Xml;
+using System.Data.SqlClient;
+
+namespace SoSlow {
+
+
+    delegate string ColumnValidator(string name, string value);
+
+    class DumpReader : MinimalDataReader {
+
+        DataTable schema;
+        DataColumn nameColumn;
+        ColumnValidator validator;
+        XmlTextReader reader;
+
+        public DumpReader(string filename, string target, SqlConnection connection, ColumnValidator validator) {
+            using (var cmd = connection.CreateCommand()) {
+                cmd.CommandType = CommandType.Text;
+                cmd.CommandText = "select top 1 * from " + target;
+                using (var reader = cmd.ExecuteReader()) {
+                    schema = reader.GetSchemaTable();
+                }
+            }
+
+            nameColumn = schema.Columns[0];
+
+            this.reader = new XmlTextReader(filename);
+            this.validator = validator;
+        }
+
+        int rowNumber;
+        public override bool Read() {
+            rowNumber++;
+            bool gotRow = false;
+            while (reader.Read()) {
+                if (reader.Name == "row") {
+                    gotRow = true;
+                    break;
+                }
+            }
+            return gotRow;
+        }
+
+        public override void Dispose() {
+            reader.Close();
+        }
+
+        public override int FieldCount {
+            get { return schema.Rows.Count; }
+        }
+
+        public override object GetValue(int i) {
+            string name = (string)schema.Rows[i][nameColumn];
+            return ValidateOrDefault(name, reader.GetAttribute(name));
+        }
+
+        private string ValidateOrDefault(string name, string data) {
+            if (validator != null) {
+                return validator(name, data);
+            }
+            return data;
+        }
+
+        public override DataTable GetSchemaTable() {
+            return schema;
+        }
+
+    } 
+}
diff --git a/Importer.cs b/Importer.cs
@@ -62,193 +62,5 @@ class Importer {
 
 
 
-        delegate string ColumnValidator(string name, string value); 
-
-        class DumpReader : IDataReader {
-
-
-
-            DataTable schema;
-            DataColumn nameColumn;
-            ColumnValidator validator; 
-            XmlTextReader reader;
-
-            public DumpReader(string filename, string target, SqlConnection connection, ColumnValidator validator) {
-                using (var cmd = connection.CreateCommand()) {
-                    cmd.CommandType = CommandType.Text;
-                    cmd.CommandText = "select top 1 * from " + target;
-                    using (var reader = cmd.ExecuteReader()) {
-                        schema = reader.GetSchemaTable();
-                    }
-                }
-
-                nameColumn = schema.Columns[0];
-
-                this.reader = new XmlTextReader(filename);
-                this.validator = validator;
-            }
-
-            #region IDataReader Members
-
-            public void Close() {
-                throw new NotImplementedException();
-            }
-
-            public int Depth {
-                get { throw new NotImplementedException(); }
-            }
-
-            public DataTable GetSchemaTable() {
-                return schema;
-            }
-
-            public bool IsClosed {
-                get { throw new NotImplementedException(); }
-            }
-
-            public bool NextResult() {
-                throw new NotImplementedException();
-            }
-
-            int rowNumber;
-            public bool Read() {
-                rowNumber++;
-                bool gotRow = false;
-                while (reader.Read()) {
-                    if (reader.Name == "row") {
-                        gotRow = true;
-                        break;
-                    }
-                }
-                return gotRow;
-            }
-
-            public int RecordsAffected {
-                get { throw new NotImplementedException(); }
-            }
-
-            #endregion
-
-            #region IDisposable Members
-
-            public void Dispose() {
-                reader.Close();
-            }
-
-            #endregion
-
-            #region IDataRecord Members
-
-            public int FieldCount {
-                get { return schema.Rows.Count; }
-            }
-
-            public bool GetBoolean(int i) {
-                throw new NotImplementedException();
-            }
-
-            public byte GetByte(int i) {
-                throw new NotImplementedException();
-            }
-
-            public long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) {
-                throw new NotImplementedException();
-            }
-
-            public char GetChar(int i) {
-                throw new NotImplementedException();
-            }
-
-            public long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) {
-                throw new NotImplementedException();
-            }
-
-            public IDataReader GetData(int i) {
-                throw new NotImplementedException();
-            }
-
-            public string GetDataTypeName(int i) {
-                throw new NotImplementedException();
-            }
-
-            public DateTime GetDateTime(int i) {
-                throw new NotImplementedException();
-            }
-
-            public decimal GetDecimal(int i) {
-                throw new NotImplementedException();
-            }
-
-            public double GetDouble(int i) {
-                throw new NotImplementedException();
-            }
-
-            public Type GetFieldType(int i) {
-                throw new NotImplementedException();
-            }
-
-            public float GetFloat(int i) {
-                throw new NotImplementedException();
-            }
-
-            public Guid GetGuid(int i) {
-                throw new NotImplementedException();
-            }
-
-            public short GetInt16(int i) {
-                throw new NotImplementedException();
-            }
-
-            public int GetInt32(int i) {
-                throw new NotImplementedException();
-            }
-
-            public long GetInt64(int i) {
-                throw new NotImplementedException();
-            }
-
-            public string GetName(int i) {
-                throw new NotImplementedException();
-            }
-
-            public int GetOrdinal(string name) {
-                throw new NotImplementedException();
-            }
-
-            public string GetString(int i) {
-                throw new NotImplementedException();
-            }
-
-            public object GetValue(int i) {
-                string name = (string)schema.Rows[i][nameColumn];  
-                return ValidateOrDefault(name, reader.GetAttribute(name));
-            }
-
-            private string ValidateOrDefault(string name, string data) {
-                if (validator != null) {
-                    return validator(name, data); 
-                }
-                return data;
-            }
-
-            public int GetValues(object[] values) {
-                throw new NotImplementedException();
-            }
-
-            public bool IsDBNull(int i) {
-                throw new NotImplementedException();
-            }
-
-            public object this[string name] {
-                get { throw new NotImplementedException(); }
-            }
-
-            public object this[int i] {
-                get { throw new NotImplementedException(); }
-            }
-
-            #endregion
-        } 
-
     }
 }
diff --git a/MainForm.cs b/MainForm.cs
@@ -72,18 +72,18 @@ public partial class MainForm : Form {
 
             SqlConnection cnn = new SqlConnection(connectionString.Text);
             cnn.Open();
-            using (var cmd = cnn.CreateCommand()) {
-                cmd.CommandText = LoadResource("SoSlow.RecreateDB.sql");
-                cmd.ExecuteNonQuery();
-            }
+            CreateDB(cnn);
 
             string[] files = new string[] { "comments", "badges", "posts", "users", "votes" };
 
             var importers = new List<Importer>();
 
             foreach (var file in files) {
                 Importer importer = new Importer(
-                    string.Format("c:\\temp\\{0}.xml",file), TitleCase(file), cnn);
+                    Path.Combine(location.Text, string.Format("{0}.xml",file)),
+                    TitleCase(file), 
+                    cnn
+                );
                 importer.Progress += new EventHandler<ProgressEventArgs>(importer_Progress);
                 importers.Add(importer);
             }
@@ -97,16 +97,42 @@ public partial class MainForm : Form {
                     importer.Import(); 
                 }
 
-
                 SetProgressMessage("Importing cc_wiki_field!");
-                ImportCCWiki(cnn); 
+                ImportCCWiki(cnn);
+
+                SetProgressMessage("Creating Tag Refs!");
+                baseProgressMessage = "Impoting tag refs";
+                ImportTagRefs(cnn); 
 
                 SetProgressMessage("Done !");
                 EnableImportButton(); 
             });
 
         }
 
+        private void ImportTagRefs(SqlConnection cnn) {
+
+            SqlBulkCopy copy = new SqlBulkCopy(cnn, SqlBulkCopyOptions.TableLock, null);
+            copy.DestinationTableName = "PostTags";
+            copy.BatchSize = 5000;
+            copy.NotifyAfter = 5000;
+            copy.SqlRowsCopied += new SqlRowsCopiedEventHandler(copy_SqlRowsCopied);
+            using (var reader = new TagReader(connectionString.Text)) {
+                copy.WriteToServer(reader);
+            }
+        }
+
+        void copy_SqlRowsCopied(object sender, SqlRowsCopiedEventArgs e) {
+            importer_Progress(this, new ProgressEventArgs() { RowsImported = (int)e.RowsCopied });
+        }
+
+        private void CreateDB(SqlConnection cnn) {
+            using (var cmd = cnn.CreateCommand()) {
+                cmd.CommandText = LoadResource("SoSlow.RecreateDB.sql");
+                cmd.ExecuteNonQuery();
+            }
+        }
+
         private void ImportCCWiki(SqlConnection cnn) {
 
             using (var cmd = cnn.CreateCommand()) {