blocks in socks

BUILD

@@ -211,4 +211,16 @@ java_test(
       ":protolib",
     ],
 )
+java_test(
+    name = "multipart_upload_test",
+    srcs = ["test/MultipartUploadTest.java"],
+    test_class = "channels.MultipartUploadTest",
+    size="medium",
+    deps = [
+      "@junit_junit//jar",
+      "@snowblossom//lib:lib",
+      ":channelslib",
+      ":protolib",
+    ],
+)
 

protolib/channels.proto

@@ -247,6 +247,7 @@ message ContentInfo {
   ContentReference parent_ref = 10;
   repeated ContentReference refs = 11;
 
+  // Just whatever extra data we want to store
   map<string, bytes> content_data_map = 12;
 
   // Things to be stored in the channel map.

src/ApiUtils.java

@@ -2,30 +2,28 @@
 
 import com.google.protobuf.ByteString;
 import java.io.InputStream;
-import java.util.Map;
 import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
 import java.util.Random;
 import java.util.TreeMap;
-import java.util.LinkedList;
 import net.minidev.json.JSONArray;
 import net.minidev.json.JSONObject;
 import net.minidev.json.parser.JSONParser;
 import snowblossom.channels.proto.*;
 import snowblossom.lib.AddressSpecHash;
 import snowblossom.lib.AddressUtil;
+import snowblossom.lib.ChainHash;
 import snowblossom.lib.DigestUtil;
 import snowblossom.lib.HexUtil;
 import snowblossom.lib.RpcUtil;
-import snowblossom.lib.AddressUtil;
-import snowblossom.lib.ChainHash;
 import snowblossom.lib.ValidationException;
-import snowblossom.proto.WalletDatabase;
-import snowblossom.client.StubHolder;
 import snowblossom.proto.AddressSpec;
 import snowblossom.proto.RequestAddress;
-import snowblossom.proto.TxOutList;
 import snowblossom.proto.TransactionOutput;
+import snowblossom.proto.TxOutList;
 import snowblossom.proto.TxOutPoint;
+import snowblossom.proto.WalletDatabase;
 
 public class ApiUtils
 {
@@ -254,6 +252,19 @@ public static void submitBlock(JSONObject input, ChannelNode node, ChannelContex
     BlockGenUtils.createBlockForContent(ctx, content,  node.getWalletDB());
   }
 
+  public static void submitFileBlock(InputStream in, ChannelNode node, ChannelContext ctx)
+    throws Exception
+  {
+    MultipartSlicer ms = new MultipartSlicer(in);
+
+    ContentInfo.Builder ci_proto = ContentInfo.newBuilder();
+
+    ci_proto.putContentDataMap("blog_entry", ByteString.copyFrom("true".getBytes()));
+
+    BlockGenUtils.createBlockForFilesMultipart(ctx, ms, node.getWalletDB(), ci_proto.build()); 
+
+  }
+
 
   public static TxOutPoint getFboOutpoint(ChannelNode node, AddressSpec address)
   {
@@ -299,7 +310,6 @@ public static JSONObject amIBlockSigner(ChannelNode node, ChannelContext ctx)
       allowed_signers.addAll(settings.getBlockSignerSpecHashesList());
       allowed_signers.addAll(settings.getAdminSignerSpecHashesList());
 
-
 		  WalletDatabase wdb = node.getWalletDB();
 
       AddressSpec addr = wdb.getAddresses(0);
@@ -314,18 +324,12 @@ public static JSONObject amIBlockSigner(ChannelNode node, ChannelContext ctx)
         reply.put("result", false);
 
       }
-
-
-
     }
     else
     {
       reply.put("result", false);
     }
 
-
-
-
     return reply;
 
   }

src/BlockGenUtils.java

@@ -4,9 +4,10 @@
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.util.List;
 import java.security.MessageDigest;
 import java.util.LinkedList;
+import java.util.List;
+import java.util.TreeMap;
 import snowblossom.channels.proto.*;
 import snowblossom.lib.AddressSpecHash;
 import snowblossom.lib.AddressUtil;
@@ -91,6 +92,81 @@ public static void createBlockForContent(ChannelContext ctx, List<SignedMessage>
 
   }
 
+  /**
+   * @param base_content_info merge this into all the created content info objects
+   */
+  public static void createBlockForFilesMultipart(ChannelContext ctx, MultipartSlicer ms, WalletDatabase admin, ContentInfo base_content_info)
+    throws ValidationException
+  {
+    List<SignedMessage> content_list = new LinkedList<>();
+
+    for(MultipartSlicer.FileData fd : ms.getFiles())
+    {
+      ContentInfo.Builder ci = ContentInfo.newBuilder();
+
+      ci.mergeFrom(base_content_info);
+
+      ci.setContentLength(fd.file_data.size());
+
+      String mime = fd.meta.get("Content-Type");
+      if (mime != null)
+      {
+        ci.setMimeType(mime);
+      } 
+      String filename = fd.meta.get("filename");
+      if (filename != null)
+      {
+        ci.putContentDataMap("filename", ByteString.copyFrom(filename.getBytes()));
+      }
+
+      MessageDigest md_part = DigestUtil.getMD();
+
+      int loc = 0;
+      int len = fd.file_data.size();
+
+      TreeMap<Integer, ByteString> chunk_map = new TreeMap<>();
+      int chunk_no =0;
+      while (loc < len)
+      {
+        int read_len = (int) Math.min( len-loc, ChannelGlobals.CONTENT_DATA_BLOCK_SIZE);
+
+        ByteString chunk_data = fd.file_data.substring(loc, read_len+loc);
+        loc += read_len;
+
+        ByteString part_hash = DigestUtil.hash(chunk_data);
+        ci.addChunkHash(part_hash);
+
+        chunk_map.put(chunk_no, chunk_data);
+        chunk_no++;
+      }
+
+      ci.setContentHash(DigestUtil.hash(fd.file_data));
+
+      ChannelValidation.validateContent(ci.build(), DigestUtil.getMD());
+
+      SignedMessage sm = 
+        ChannelSigUtil.signMessage( admin.getAddresses(0), admin.getKeys(0),
+          SignedMessagePayload.newBuilder().setContentInfo(ci.build()).build());
+
+      content_list.add(sm);
+
+      for(int c : chunk_map.keySet())
+      {
+        ctx.block_ingestor.ingestChunk(
+          ContentChunk.newBuilder()
+            .setMessageId(sm.getMessageId())
+            .setChunk(c) 
+            .setChunkData(chunk_map.get(c))
+            .build()
+          ,true, ci.build());
+      }
+    }
+
+    // once we are done playing with files, call 
+    createBlockForContent(ctx, content_list, admin);
+
+  }
+
   /**
    * Creates a block for the files in the directory and broadcasts it to the channel
    */ 

src/ChannelChunkGetter.java

@@ -9,10 +9,9 @@
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Semaphore;
+import java.util.logging.Logger;
 import snowblossom.channels.proto.*;
 import snowblossom.lib.ChainHash;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 public class ChannelChunkGetter extends PeriodicThread
 {

src/ChannelGlobals.java

@@ -2,7 +2,7 @@
 
 public class ChannelGlobals
 {
-  public static final String VERSION = "dev.10.23.3";
+  public static final String VERSION = "dev.10.25.0";
 
   public static final String NODE_ADDRESS_STRING="node";
   public static final String CHANNEL_ADDRESS_STRING="chan";
@@ -62,3 +62,6 @@ public class ChannelGlobals
 
 
 
+
+
+

src/ChannelNode.java

@@ -19,9 +19,9 @@
 import snowblossom.channels.*;
 import snowblossom.channels.proto.*;
 import snowblossom.channels.proto.StargateServiceGrpc.StargateServiceBlockingStub;
-import snowblossom.client.WalletUtil;
-import snowblossom.client.StubUtil;
 import snowblossom.client.StubHolder;
+import snowblossom.client.StubUtil;
+import snowblossom.client.WalletUtil;
 import snowblossom.lib.*;
 import snowblossom.lib.AddressSpecHash;
 import snowblossom.lib.db.rocksdb.JRocksDB;

src/ChannelOutsiderSender.java

@@ -1,14 +1,14 @@
 package snowblossom.channels;
 
 import com.google.protobuf.ByteString;
-import snowblossom.lib.ValidationException;
 import duckutil.PeriodicThread;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Random;
 import java.util.Set;
 import snowblossom.channels.proto.*;
+import snowblossom.lib.ValidationException;
 
 public class ChannelOutsiderSender extends PeriodicThread
 {

src/Mimer.java

@@ -572,3 +572,6 @@ public static String guessContentType(String path)
 
 
 
+
+
+