Permalink
Browse files

new index object

  • Loading branch information...
hbons committed Aug 1, 2011
1 parent 7a791ad commit 3510b85d6514f5b96fcd0c33be0914451e3e5b33
Showing with 155 additions and 91 deletions.
  1. +46 −32 Chunker.cs
  2. +66 −7 Repository.cs → Index.cs
  3. +6 −37 Main.cs
  4. +1 −1 Rainbows.csproj
  5. +36 −14 TransferManager.cs
View
@@ -16,6 +16,7 @@
using System;
+using System.Collections.Generic;
using System.IO;
namespace Rainbows {
@@ -27,6 +28,9 @@ public class Chunker {
public delegate void ChunkCreatedHandler (string chunk_file_path, int chunk_size, string chunk_hash);
public event ChunkCreatedHandler ChunkCreated;
+ public delegate void ChunkingFinishedHandler (string [] chunk_paths);
+ public event ChunkingFinishedHandler ChunkingFinished;
+
private Cryptographer cryptographer;
@@ -53,48 +57,58 @@ private void Init (string output_directory)
}
- public void FileToChunks (string source_file_path)
+ public void FileToChunks (string [] source_file_paths)
{
int chunk_size = 1024 * 1024 * 4;
+ List <string> chunk_paths = new List<string> ();
- using (FileStream stream = File.OpenRead (source_file_path))
- {
- int current_chunk_size = 0;
- var buffer = new byte [chunk_size];
- int chunk_number = 1;
+ // TODO: threads
+ foreach (string source_file_path in source_file_paths) {
- while ((current_chunk_size = stream.Read (buffer, 0, buffer.Length)) > 0)
+ using (FileStream stream = File.OpenRead (source_file_path))
{
- string hash = Cryptographer.SHA1 (buffer);
- string chunk_file_name = hash.Substring (2);
- string chunk_container = hash.Substring (0, 2);
- string chunk_container_path = Path.Combine (OutputDirectory, chunk_container);
- string chunk_file_path = Path.Combine (chunk_container_path, chunk_file_name);
-
- if (!File.Exists (chunk_file_path)) {
- if (!Directory.Exists (chunk_container_path))
- Directory.CreateDirectory (chunk_container_path);
-
- if (this.cryptographer != null) {
- byte [] crypto_buffer = this.cryptographer.Encrypt (buffer);
- File.WriteAllBytes (chunk_file_path, crypto_buffer);
-
+ int current_chunk_size = 0;
+ var buffer = new byte [chunk_size];
+ int chunk_number = 1;
+
+ while ((current_chunk_size = stream.Read (buffer, 0, buffer.Length)) > 0)
+ {
+ string hash = Cryptographer.SHA1 (buffer);
+ string chunk_file_name = hash.Substring (2);
+ string chunk_container = hash.Substring (0, 2);
+ string chunk_container_path = Path.Combine (OutputDirectory, chunk_container);
+ string chunk_file_path = Path.Combine (chunk_container_path, chunk_file_name);
+
+ if (!File.Exists (chunk_file_path)) {
+ if (!Directory.Exists (chunk_container_path))
+ Directory.CreateDirectory (chunk_container_path);
+
+ if (this.cryptographer != null) {
+ byte [] crypto_buffer = this.cryptographer.Encrypt (buffer);
+ File.WriteAllBytes (chunk_file_path, crypto_buffer);
+
+ } else {
+ File.WriteAllBytes (chunk_file_path, buffer);
+ }
+
+ chunk_paths.Add (chunk_file_path);
+
+ if (ChunkCreated != null)
+ ChunkCreated (chunk_file_path, current_chunk_size, hash);
+
+ Console.WriteLine ("Chunk " + hash + " created");
+
} else {
- File.WriteAllBytes (chunk_file_path, buffer);
+ Console.WriteLine ("Chunk " + hash + " exists");
}
-
- if (ChunkCreated != null)
- ChunkCreated (chunk_file_path, current_chunk_size, hash);
-
- Console.WriteLine ("Chunk " + hash + " created");
-
- } else {
- Console.WriteLine ("Chunk " + hash + " exists");
+
+ chunk_number++;
}
-
- chunk_number++;
}
}
+
+ if (ChunkingFinished != null)
+ ChunkingFinished (chunk_paths.ToArray ());
}
View
@@ -22,35 +22,92 @@
namespace Rainbows {
- public class Repository {
+ public class Index {
- public Repository (string path)
+ public readonly string DatabasePath;
+ public readonly string CheckoutPath;
+
+
+ public Index (string database_path, string checkout_path)
+ {
+ DatabasePath = database_path;
+ CheckoutPath = checkout_path;
+ }
+
+
+ public void Status ()
{
}
public void Commit ()
{
- // - Walk the new tree and create blobs, trees
- // - Create new commit object: 'current' file is ParentHash,
+ Chunker chunker = new Chunker (DatabasePath,
+ new Cryptographer ("cGFzc3dvcmQAAAAAAAAAAA=="));
+
+ TransferManager transfer_manager = new TransferManager (
+ DatabasePath, "/Users/hbons/rsync-test");
+
+ chunker.ChunkCreated += delegate (string chunk_file_path, int chunk_size,
+ string chunk_hash) {
+
+ transfer_manager.QueueUpload ();
+ };
+
+ chunker.ChunkingFinished += delegate {
+ // - Walk the new tree and create blobs, trees
+
+ };
+
+ chunker.FileToChunks (new string [] {"/Users/hbons/hp2.avi"});
+
+
// - Update 'current' file
}
- public void Push ()
+ public void Checkout (string commit_hash)
{
}
- public void Pull ()
+ public bool Push ()
{
+ return true;
+ }
+
+
+ public void PullAndRebase ()
+ {
+ // TODO: 1. Get the latest HEAD commit from the server
+ // 2. Parse the tree and download all the objects we don't have
+ TransferManager transfer_manager = new TransferManager (
+ DatabasePath, "/Users/hbons/rsync-test");
+
+ string [] new_remote_objects = new string [0];
+ transfer_manager.DownloadObjects (new_remote_objects);
+ }
+
+
+ public static Index Init (string path)
+ {
+ if (!Directory.Exists (path))
+ Directory.CreateDirectory (path);
+
+ string database_path = Path.Combine (path, ".sparkleshare");
+
+ if (!Directory.Exists (database_path))
+ Directory.CreateDirectory (database_path);
+
+ return new Index (database_path, path);
}
}
- // TODO: integrate into Repository
+
+ // TODO: integrate into Index
public class Blobs {
public readonly string OutputDirectory;
@@ -67,6 +124,7 @@ public Blobs (string output_directory)
public void Store (string file_name, string [] chunk_hashes)
{
+ // TODO: we really need the file hash
string hash = string.Join ("", chunk_hashes);
string file_store_name = Cryptographer.SHA1 (Encoding.ASCII.GetBytes (hash));
@@ -84,6 +142,7 @@ public void Store (string file_name, string [] chunk_hashes)
byte [] buffer = Encoding.ASCII.GetBytes (chunk_hash + "\n");
stream.Write (buffer, 0, buffer.Length);
}
+
Console.WriteLine ("Created: " + file_store_path);
}
}
View
43 Main.cs
@@ -22,47 +22,16 @@
using System.Text;
using System.Xml;
-namespace Rainbows
-{
- public class Rainbows
- {
- public static string RemoteUrl = "ssh://localhost:/Users/hbons/TestTest";
- public static string LocalPath = "/Users/hbons/SparkleShare/Rainbows/.sparkleshare";
+namespace Rainbows {
+
+ public class Rainbows {
public static void Main (string [] args)
{
- //
- // PLAYING GROUND
- //
-
- Stopwatch s = new Stopwatch ();
- s.Start ();
-
- Chunker chunker = new Chunker ("/Users/hbons/SparkleShare/Rainbows/.sparkleshare",
- new Cryptographer ("cGFzc3dvcmQAAAAAAAAAAA=="));
-
- List<string> chunk_paths = new List<string> ();
- List<string> hashes = new List<string> ();
-
- TransferManager transfer_manager = new TransferManager (LocalPath + "/", "/Users/hbons/rsync-test");
-
- chunker.ChunkCreated += delegate (string chunk_file_path, int chunk_size, string chunk_hash) {
- // Console.WriteLine ("Created: " + chunk_file_path + " (" + chunk_size + " bytes)");
- chunk_paths.Add (chunk_file_path);
- hashes.Add (chunk_hash);
-
-
- transfer_manager.QueueUpload ();
- };
-
- chunker.FileToChunks ("/Users/hbons/hp2.avi");
-
- // Blobs blobs = new Blobs ("Users/hbons/SparkleShare/Rainbows/.sparkleshare");
- //blobs.Store ("hp.avi", hashes.ToArray ());
-
+ Index index = new Index ("/Users/hbons/SparkleShare/Rainbows/.sparkleshare",
+ "/Users/hbons/SparkleShare/Rainbows");
- s.Stop ();
- Console.WriteLine ("Total time: " + (int) s.Elapsed.TotalSeconds + " seconds");
+ index.Status ();
}
}
}
View
@@ -37,8 +37,8 @@
<Compile Include="Cryptographer.cs" />
<Compile Include="Chunker.cs" />
<Compile Include="Objects.cs" />
- <Compile Include="Repository.cs" />
<Compile Include="TransferManager.cs" />
+ <Compile Include="Index.cs" />
</ItemGroup>
<ItemGroup>
<Reference Include="System" />
View
@@ -17,40 +17,59 @@
using System;
using System.Diagnostics;
+using System.IO;
using System.Threading;
namespace Rainbows {
public class TransferManager {
- private bool queued_upload = false;
- private bool busy = false;
- private Process rsync_process = new Process ();
+ public delegate void UploadFinishedHandler ();
+ public event UploadFinishedHandler UploadFinished;
+
+ private bool queued_upload = false;
+ private bool busy = false;
+ private Process upload_process = new Process ();
+ private Process download_process = new Process ();
public TransferManager (string path, string remote_path)
{
- this.rsync_process.StartInfo.FileName = "rsync";
- this.rsync_process.StartInfo.Arguments = "--ignore-existing --bwlimit=500 " +
+ if (!path.EndsWith ("" + Path.DirectorySeparatorChar))
+ path += Path.DirectorySeparatorChar;
+
+ this.upload_process.EnableRaisingEvents = true;
+ this.upload_process.StartInfo.RedirectStandardOutput = false;
+ this.upload_process.StartInfo.UseShellExecute = false;
+ this.upload_process.StartInfo.WorkingDirectory = path;
+ this.upload_process.StartInfo.FileName = "rsync";
+ this.upload_process.StartInfo.Arguments = "--ignore-existing " +
+ "--bwlimit=1000 --recursive --whole-file --progress --exclude=HEAD " + path +
+ " " + remote_path;
+
+ this.download_process.EnableRaisingEvents = true;
+ this.download_process.StartInfo.RedirectStandardOutput = false;
+ this.download_process.StartInfo.UseShellExecute = false;
+ this.download_process.StartInfo.WorkingDirectory = path;
+ this.download_process.StartInfo.FileName = "rsync";
+ this.download_process.StartInfo.Arguments = "--ignore-existing --bwlimit=1000 " +
"--recursive --whole-file --progress " + path + " " + remote_path;
-
- Console.WriteLine (this.rsync_process.StartInfo.Arguments);
-
- this.rsync_process.EnableRaisingEvents = true;
- this.rsync_process.StartInfo.RedirectStandardOutput = false;
- this.rsync_process.StartInfo.UseShellExecute = false;
- this.rsync_process.StartInfo.WorkingDirectory = path;
}
public void UploadObjects ()
{
this.busy = true;
- this.rsync_process.Start ();
- this.rsync_process.WaitForExit ();
+ this.upload_process.Start ();
+ this.upload_process.WaitForExit ();
this.busy = false;
+
+ if (this.queued_upload) {
+ this.queued_upload = false;
+ UploadObjects ();
+ }
}
@@ -66,6 +85,9 @@ public void QueueUpload ()
);
thread.Start ();
+
+ } else {
+ this.queued_upload = true;
}
}

0 comments on commit 3510b85

Please sign in to comment.