Skip to content
This repository
Browse code

The basic function of the storage facility is now done.

  • Loading branch information...
commit e8d212b166aeaf25fc529ce2be7e00eb933ca3f6 1 parent 8dfad5f
Mikael Norrman authored
2  datastorageproject/log.txt
... ... @@ -0,0 +1,2 @@
  1 +Tjohojava.lang.ArrayIndexOutOfBoundsException: 4
  2 + at com.mnorrman.datastorageproject.LoggerWriter.main(LoggerWriter.java:42)
105 datastorageproject/src/com/mnorrman/datastorageproject/BackStorage.java
@@ -4,10 +4,16 @@
4 4 */
5 5 package com.mnorrman.datastorageproject;
6 6
  7 +import com.mnorrman.datastorageproject.objects.IndexedDataObject;
  8 +import com.mnorrman.datastorageproject.tools.Checksum;
  9 +import com.mnorrman.datastorageproject.tools.HexConverter;
7 10 import java.io.File;
8 11 import java.io.IOException;
9 12 import java.io.RandomAccessFile;
  13 +import java.nio.ByteBuffer;
10 14 import java.nio.channels.FileChannel;
  15 +import java.util.Iterator;
  16 +import java.util.LinkedList;
11 17
12 18 /**
13 19 *
@@ -34,9 +40,95 @@ public BackStorage initialize() throws IOException{
34 40 return this;
35 41 }
36 42
  43 + public LinkedList<IndexedDataObject> reindexData(){
  44 + LinkedList<IndexedDataObject> list = new LinkedList<IndexedDataObject>();
  45 + long position = 0;
  46 + ByteBuffer data = ByteBuffer.allocateDirect(512);
  47 + try{
  48 + FileChannel channel = fileConnection.getChannel();
  49 + channel.position(position);
  50 + do{
  51 + data.clear(); //Clear it first
  52 + channel.read(data);
  53 + data.rewind();
  54 +
  55 + byte[] temp;
  56 + byte[] checksum = new byte[16];
  57 + String colname, rowname, owner;
  58 + long version, len;
  59 +
  60 + temp = new byte[128];
  61 + data.get(temp);
  62 + colname = new String(temp).trim();
  63 +
  64 + temp = new byte[128];
  65 + data.get(temp);
  66 + rowname = new String(temp).trim();
  67 +
  68 + version = data.getLong();
  69 +
  70 + len = data.getLong();
  71 +
  72 + data.get(checksum);
  73 +
  74 + temp = new byte[128];
  75 + data.get(temp);
  76 + owner = new String(temp).trim();
  77 +
  78 + list.add(new IndexedDataObject(colname, rowname, owner, 0L, len, version, checksum));
  79 + position += (512+len);
  80 + channel.position(position);
  81 + }while(channel.size() - channel.position() > 0);
  82 + }catch(IOException e){
  83 + e.printStackTrace();
  84 + }
  85 + return list;
  86 + }
  87 +
37 88 public boolean performIntegrityCheck(){
38   -
39   - return false;
  89 + long position = 0;
  90 + ByteBuffer length = ByteBuffer.allocate(8);
  91 + ByteBuffer checksum = ByteBuffer.allocate(16);
  92 + ByteBuffer data;
  93 + try{
  94 + FileChannel channel = fileConnection.getChannel();
  95 + channel.position(position);
  96 + do{
  97 + length.clear();
  98 + checksum.clear(); //Clear it first
  99 + channel.position(position + 264); //Jump to length and checksum
  100 + channel.read(length);
  101 + channel.read(checksum);
  102 + length.rewind();
  103 + checksum.rewind();
  104 +
  105 + //Set variables
  106 + long len;
  107 + byte[] checksumDataFromIndex = new byte[16];
  108 +
  109 + //Read from bytebuffers
  110 + len = length.getLong();
  111 + checksum.get(checksumDataFromIndex);
  112 +
  113 + data = ByteBuffer.allocate((int)len);
  114 + channel.position(position + 512);
  115 + channel.read(data);
  116 +
  117 + data.flip();
  118 +
  119 + byte[] checksumDataFromData = Checksum.getFor(data);
  120 +
  121 + if(!HexConverter.toHex(checksumDataFromIndex).equals(HexConverter.toHex(checksumDataFromData))){
  122 + return false;
  123 + }
  124 +
  125 + position += (512+len);
  126 + channel.position(position);
  127 + }while(channel.size() - channel.position() > 0);
  128 + }catch(IOException e){
  129 + e.printStackTrace();
  130 + }
  131 + return true;
40 132 }
41 133
42 134 public FileChannel getChannel(){
@@ -46,7 +138,14 @@ public FileChannel getChannel(){
46 138 public static void main(String[] args) {
47 139 BackStorage t = new BackStorage();
48 140 try{
49   - t.initialize();
  141 + LinkedList<IndexedDataObject> list = t.initialize().reindexData();
  142 + System.out.println("List = " + list.size());
  143 + Iterator<IndexedDataObject> it = list.iterator();
  144 + while(it.hasNext()){
  145 + System.out.println(it.next().toString());
  146 + }
  147 +
  148 + System.out.println("Integrity check is: " + t.performIntegrityCheck());
50 149 }catch(IOException e){
51 150 e.printStackTrace();
52 151 }
110 datastorageproject/src/com/mnorrman/datastorageproject/DataProcessor.java
@@ -5,6 +5,7 @@
5 5 package com.mnorrman.datastorageproject;
6 6
7 7 import com.mnorrman.datastorageproject.objects.IndexedDataObject;
  8 +import com.mnorrman.datastorageproject.objects.UnindexedDataObject;
8 9 import com.mnorrman.datastorageproject.tools.Checksum;
9 10 import java.io.IOException;
10 11 import java.nio.ByteBuffer;
@@ -36,108 +37,43 @@ public ByteBuffer retrieveData(IndexedDataObject ido){
36 37 }
37 38 return null;
38 39 }
39   -
40   - public IndexedDataObject retrieveMetaData(long offset, long length){
41   - try{
42   - ByteBuffer data = ByteBuffer.allocateDirect(512 + (int)length);
43   - dataChannel.position(offset);
44   - dataChannel.read(data);
45   - data.rewind();
46   -
47   - byte[] temp;
48   - byte[] checksum = new byte[16];
49   - String colname, rowname, owner;
50   - long version, len;
51   -
52   - temp = new byte[128];
53   - data.get(temp);
54   - colname = new String(temp).trim();
55   -
56   - temp = new byte[128];
57   - data.get(temp);
58   - rowname = new String(temp).trim();
59   -
60   - version = data.getLong();
61   -
62   - len = data.getLong();
63   -
64   - data.get(checksum);
65   -
66   - temp = new byte[128];
67   - data.get(temp);
68   - owner = new String(temp).trim();
69   -
70   - //return new IndexedDataObject(colname, rowname, owner, offset, length, version, checksum)
71   - }catch(IOException e){
72   - Logger.getGlobal().log(Level.SEVERE, e.getMessage());
73   - }
74   - return null;
75   - }
76   -
77   - public void testReadData(){
78   - try{
79   - ByteBuffer buff = ByteBuffer.allocateDirect(512);
80   - byte[] temp;
81   - dataChannel.position(0);
82   - dataChannel.read(buff);
83   - buff.flip();
84 40
85   - temp = new byte[128];
86   - buff.get(temp);
87   - System.out.println("Value 1: " + new String(temp).trim());
88   -
89   - temp = new byte[128];
90   - buff.get(temp);
91   - System.out.println("Value 2: " + new String(temp).trim());
92   -
93   - System.out.println("Value 3: " + buff.getLong());
94   - System.out.println("Value 4: " + buff.getLong());
95   -
96   - temp = new byte[16];
97   - buff.get(temp);
98   - System.out.println("Value 5: " + new String(temp));
99   -
100   - temp = new byte[128];
101   - buff.get(temp);
102   - System.out.println("Value 6: " + new String(temp).trim());
103   -
104   - buff.clear();
105   - }catch(IOException e){
106   - e.printStackTrace();
107   - }
108   - }
109   -
110   - public boolean storeData(String colname, String rowname, long version, long length, String owner, ByteBuffer data){
  41 + public IndexedDataObject storeData(UnindexedDataObject udo){
111 42 try{
112 43 ByteBuffer buffer = ByteBuffer.allocateDirect(512); //416 + 96 (Void)
113 44 byte[] colnameBytes = new byte[128];
114 45 byte[] rownameBytes = new byte[128];
115 46 byte[] ownerBytes = new byte[128];
116   - System.arraycopy(colname.getBytes(), 0, colnameBytes, 0, colname.getBytes().length);
117   - System.arraycopy(rowname.getBytes(), 0, rownameBytes, 0, rowname.getBytes().length);
118   - System.arraycopy(owner.getBytes(), 0, ownerBytes, 0, owner.getBytes().length);
  47 + System.arraycopy(udo.getColname().getBytes(), 0, colnameBytes, 0, udo.getColname().getBytes().length);
  48 + System.arraycopy(udo.getRowname().getBytes(), 0, rownameBytes, 0, udo.getRowname().getBytes().length);
  49 + System.arraycopy(udo.getOwner().getBytes(), 0, ownerBytes, 0, udo.getOwner().getBytes().length);
  50 +
  51 + long newVersion = System.currentTimeMillis();
119 52
120 53 buffer.put(colnameBytes);
121 54 buffer.put(rownameBytes);
122   - buffer.putLong(version);
123   - buffer.putLong(length);
124   - buffer.put(Checksum.getFor(data));
  55 + buffer.putLong(newVersion);
  56 + buffer.putLong(udo.getLength());
  57 + buffer.put(Checksum.getFor(udo.getData()));
125 58 buffer.put(ownerBytes);
126 59
127 60 byte[] voidData = new byte[96];
128 61 buffer.put(voidData);
129 62 buffer.flip();
130   - data.flip();
  63 + udo.getData().flip();
  64 +
  65 + long newOffset = dataChannel.size();
  66 +
131 67 dataChannel.position(dataChannel.size());
132 68 dataChannel.write(buffer);
133   - dataChannel.write(data);
134   - return true;
  69 + dataChannel.write(udo.getData());
  70 + return new IndexedDataObject(udo, newOffset, newVersion);
135 71
136 72 }catch(IOException e){
137 73 Logger.getGlobal().log(Level.SEVERE, e.getMessage());
138 74 e.printStackTrace();
139 75 }
140   - return false;
  76 + return null;
141 77 }
142 78
143 79 public static void main(String[] args) {
@@ -145,19 +81,15 @@ public static void main(String[] args) {
145 81 try{
146 82 b.initialize();
147 83 }catch(IOException e){ e.printStackTrace(); }
  84 +
148 85 DataProcessor p = new DataProcessor(b.getChannel());
149 86 ByteBuffer buf = ByteBuffer.allocate(10);
150 87 buf.put(new byte[]{ 110, 111, 112, 113, 114, 115, 116, 117, 118, 119 });
151   - p.storeData("Testvalue a", "Testvalue b", System.currentTimeMillis(), 10L, "Ownervalue", buf);
152   -
153   - p.testReadData();
154 88
155   - System.out.println("Testing to retrieve teh data!!!!");
156   - IndexedDataObject o = new IndexedDataObject("Testvalue a", "Testvalue b", "ownervalue", 0, 10, 1L, Checksum.getFor(buf));
  89 + UnindexedDataObject udo = new UnindexedDataObject(buf, "This is the column", "This is the row", "Mikael Norrman");
  90 + IndexedDataObject ido = p.storeData(udo);
157 91
158   - byte[] stringData = new byte[(int)o.getLength()];
159   - p.retrieveData(o).get(stringData);
160   - System.out.println(new String(stringData).trim());
  92 + System.out.println(ido);
161 93 }
162 94
163 95 }
42 datastorageproject/src/com/mnorrman/datastorageproject/Index.java
@@ -6,27 +6,59 @@
6 6
7 7 import com.mnorrman.datastorageproject.objects.IndexedDataObject;
8 8 import com.mnorrman.datastorageproject.tools.Checksum;
  9 +import java.util.ArrayList;
9 10 import java.util.HashMap;
  11 +import java.util.Iterator;
  12 +import java.util.List;
10 13
11 14 /**
12 15 *
13 16 * @author Mikael
14 17 */
15   -public class Index extends HashMap<String, IndexedDataObject> {
  18 +public class Index extends HashMap<String, ArrayList<IndexedDataObject>> {
16 19
17 20 public Index(){
18 21 super();
19 22 }
20 23
21   - public void insertNew(IndexedDataObject ido){
22   - put(Checksum.getFor(ido.getColname(), ido.getRowname()), ido);
  24 + public void insertIndex(IndexedDataObject ido){
  25 + String checksum = Checksum.getFor(ido.getColname(), ido.getRowname());
  26 + if(containsKey(checksum)){
  27 + ArrayList<IndexedDataObject> temp = get(checksum);
  28 + IndexedDataObject idoTemp = null;
  29 + temp.add(ido);
  30 + for(int a = temp.size() - 1; a > 0; a--){
  31 + if(temp.get(a).getVersion() > temp.get(a-1).getVersion()){
  32 + idoTemp = temp.get(a-1);
  33 + temp.set(a-1, temp.get(a));
  34 + temp.set(a, idoTemp);
  35 + }else{
  36 + break;
  37 + }
  38 + }
  39 + }else{
  40 + put(checksum, new ArrayList<IndexedDataObject>());
  41 + get(checksum).add(ido);
  42 + }
  43 + }
  44 +
  45 + public IndexedDataObject get(String a, String b){
  46 + return get(Checksum.getFor(a, b)).get(0); //Zero is head
23 47 }
24 48
25   - public IndexedDataObject getIndexedObject(String hash){
26   - return get(hash);
  49 + public int getNumberOfVersions(IndexedDataObject ido){
  50 + return get(Checksum.getFor(ido.getColname(), ido.getRowname())).size();
27 51 }
28 52
29 53 public boolean doIndexedObjectExist(String hash){
30 54 return containsKey(hash);
31 55 }
  56 +
  57 + public void insertAll(List<IndexedDataObject> list){
  58 + Iterator<IndexedDataObject> iterator = list.iterator();
  59 + while(iterator.hasNext()){
  60 + IndexedDataObject temp = iterator.next();
  61 + insertIndex(temp);
  62 + }
  63 + }
32 64 }
47 datastorageproject/src/com/mnorrman/datastorageproject/LoggerWriter.java
... ... @@ -0,0 +1,47 @@
  1 +/*
  2 + * To change this template, choose Tools | Templates
  3 + * and open the template in the editor.
  4 + */
  5 +package com.mnorrman.datastorageproject;
  6 +
  7 +import java.io.*;
  8 +
  9 +/**
  10 + *
  11 + * @author Mikael
  12 + */
  13 +public class LoggerWriter extends FilterWriter{
  14 +
  15 + public LoggerWriter(Writer arg0) {
  16 + super(arg0);
  17 + }
  18 +
  19 + @Override
  20 + public void write(String arg0) throws IOException {
  21 + super.write(arg0);
  22 + System.out.println(arg0);
  23 + flush();
  24 + }
  25 +
  26 + @Override
  27 + public void write(int arg0) throws IOException {
  28 + super.write(arg0);
  29 + System.out.print("" + (char)arg0);
  30 + }
  31 +
  32 +
  33 +
  34 +
  35 +
  36 + public static void main(String[] args) throws IOException{
  37 + PrintWriter pw = new PrintWriter(new LoggerWriter(new FileWriter(new File("log.txt"))));
  38 +
  39 + pw.write("Tjoho");
  40 + try{
  41 + int[] a = new int[1];
  42 + int b = a[4];
  43 + }catch(IndexOutOfBoundsException e){
  44 + e.printStackTrace(pw);
  45 + }
  46 + }
  47 +}
46 datastorageproject/src/com/mnorrman/datastorageproject/Main.java
@@ -4,16 +4,60 @@
4 4 */
5 5 package com.mnorrman.datastorageproject;
6 6
  7 +import com.mnorrman.datastorageproject.objects.IndexedDataObject;
  8 +import com.mnorrman.datastorageproject.objects.UnindexedDataObject;
  9 +import com.mnorrman.datastorageproject.tools.Checksum;
  10 +import java.io.File;
  11 +import java.io.FileInputStream;
  12 +import java.io.IOException;
  13 +import java.nio.ByteBuffer;
  14 +import java.nio.channels.FileChannel;
  15 +
7 16 /**
8 17 *
9 18 * @author Mikael
10 19 */
11 20 public class Main {
12 21
  22 + public static Index localIndex;
  23 + public BackStorage storage;
  24 +
  25 + public Main(){
  26 + try{
  27 + storage = new BackStorage().initialize();
  28 + DataProcessor dp = new DataProcessor(storage.getChannel());
  29 +
  30 + File file = new File("testfile.exe");
  31 + ByteBuffer dataBuffer = ByteBuffer.allocate((int)file.length());
  32 + FileChannel fc = new FileInputStream(file).getChannel();
  33 + fc.read(dataBuffer);
  34 + fc.close();
  35 + UnindexedDataObject udo = new UnindexedDataObject(dataBuffer, "Example1", "Example1-2", "Mikael Norrman");
  36 +
  37 + localIndex.insertIndex(dp.storeData(udo));
  38 +
  39 + IndexedDataObject ido = localIndex.get("Example1", "Example1-2");
  40 + System.out.println(ido);
  41 +
  42 + }catch(IOException e){
  43 + e.printStackTrace();
  44 + }
  45 + /*try{
  46 + storage = new BackStorage().initialize();
  47 + localIndex.insertAll(storage.reindexData());
  48 +
  49 + System.out.println("Local index has: " + localIndex.size());
  50 + System.out.println("Number of versions for first: " + localIndex.getNumberOfVersions(localIndex.get("Example1", "Example1-2")));
  51 + }catch(IOException e){
  52 + e.printStackTrace();
  53 + }*/
  54 + }
  55 +
13 56 /**
14 57 * @param args the command line arguments
15 58 */
16 59 public static void main(String[] args) {
17   - // TODO code application logic here
  60 + localIndex = new Index();
  61 + Main m = new Main();
18 62 }
19 63 }
52 datastorageproject/src/com/mnorrman/datastorageproject/objects/DataObject.java
... ... @@ -0,0 +1,52 @@
  1 +/*
  2 + * To change this template, choose Tools | Templates
  3 + * and open the template in the editor.
  4 + */
  5 +package com.mnorrman.datastorageproject.objects;
  6 +
  7 +import java.io.Serializable;
  8 +
  9 +/**
  10 + *
  11 + * @author Mikael
  12 + */
  13 +public abstract class DataObject implements Serializable{
  14 +
  15 + protected String colname, rowname, owner;
  16 + protected long length;
  17 + protected byte[] checksum; //always 16 byte
  18 +
  19 + public DataObject(){ }
  20 +
  21 + public DataObject(String colname, String rowname, String owner, long length, byte[] checksum) {
  22 + this.colname = colname;
  23 + this.rowname = rowname;
  24 + this.owner = owner;
  25 + this.length = length;
  26 + this.checksum = checksum;
  27 + }
  28 +
  29 + public byte[] getChecksum() {
  30 + return checksum;
  31 + }
  32 +
  33 + public String getColname() {
  34 + return colname;
  35 + }
  36 +
  37 + public long getLength() {
  38 + return length;
  39 + }
  40 +
  41 + public String getOwner() {
  42 + return owner;
  43 + }
  44 +
  45 + public String getRowname() {
  46 + return rowname;
  47 + }
  48 +
  49 + @Override
  50 + public abstract String toString();
  51 +
  52 +}
42 datastorageproject/src/com/mnorrman/datastorageproject/objects/IndexedDataObject.java
@@ -4,17 +4,16 @@
4 4 */
5 5 package com.mnorrman.datastorageproject.objects;
6 6
  7 +import com.mnorrman.datastorageproject.tools.HexConverter;
7 8 import java.io.Serializable;
8 9
9 10 /**
10 11 *
11 12 * @author Mikael
12 13 */
13   -public final class IndexedDataObject implements Serializable{
  14 +public final class IndexedDataObject extends DataObject implements Serializable{
14 15
15   - private String colname, rowname, owner;
16   - private long offset, length, version;
17   - private byte[] checksum; //always 16 byte
  16 + private long offset, version;
18 17
19 18 public IndexedDataObject(String colname, String rowname, String owner, long offset, long length, long version, byte[] checksum) {
20 19 this.colname = colname;
@@ -26,35 +25,30 @@ public IndexedDataObject(String colname, String rowname, String owner, long offs
26 25 this.checksum = checksum;
27 26 }
28 27
  28 + public IndexedDataObject(UnindexedDataObject udo, long offset, long version){
  29 + this.colname = udo.getColname();
  30 + this.rowname = udo.getRowname();
  31 + this.owner = udo.getOwner();
  32 + this.offset = offset;
  33 + this.length = udo.getLength();
  34 + this.version = version;
  35 + this.checksum = udo.getChecksum();
  36 + }
  37 +
29 38 public IndexedDataObject(){
30 39 //Should probably be removed later on.
31 40 }
32 41
33   - public byte[] getChecksum() {
34   - return checksum;
35   - }
36   -
37   - public String getColname() {
38   - return colname;
39   - }
40   -
41   - public long getLength() {
42   - return length;
43   - }
44   -
45 42 public long getOffset() {
46 43 return offset;
47 44 }
48 45
49   - public String getOwner() {
50   - return owner;
51   - }
52   -
53   - public String getRowname() {
54   - return rowname;
55   - }
56   -
57 46 public long getVersion() {
58 47 return version;
59 48 }
  49 +
  50 + @Override
  51 + public String toString() {
  52 + return "IndexedDataObject: colname=" + colname + ", rowname=" + rowname + ", owner=" + owner + ", offset=" + offset + ", length=" + length + ", version=" + version + ", checksum=" + HexConverter.toHex(checksum);
  53 + }
60 54 }
36 datastorageproject/src/com/mnorrman/datastorageproject/objects/UnindexedDataObject.java
... ... @@ -0,0 +1,36 @@
  1 +/*
  2 + * To change this template, choose Tools | Templates
  3 + * and open the template in the editor.
  4 + */
  5 +package com.mnorrman.datastorageproject.objects;
  6 +
  7 +import com.mnorrman.datastorageproject.tools.Checksum;
  8 +import com.mnorrman.datastorageproject.tools.HexConverter;
  9 +import java.nio.ByteBuffer;
  10 +
  11 +/**
  12 + *
  13 + * @author Mikael
  14 + */
  15 +public class UnindexedDataObject extends DataObject{
  16 +
  17 + private ByteBuffer data;
  18 +
  19 + public UnindexedDataObject(ByteBuffer data, String colname, String rowname, String owner) {
  20 + super(colname, rowname, owner, data.capacity(), Checksum.getFor(data));
  21 + this.data = data;
  22 + }
  23 +
  24 + public UnindexedDataObject(ByteBuffer data){
  25 + this.data = data;
  26 + }
  27 +
  28 + public ByteBuffer getData(){
  29 + return data;
  30 + }
  31 +
  32 + @Override
  33 + public String toString() {
  34 + return "UnindexedDataObject: colname=" + colname + ", rowname=" + rowname + ", owner=" + owner + ", length=" + length + ", checksum=" + HexConverter.toHex(checksum);
  35 + }
  36 +}
4 datastorageproject/src/com/mnorrman/datastorageproject/tools/Checksum.java
@@ -18,8 +18,11 @@ private Checksum(){ }
18 18
19 19 public synchronized static byte[] getFor(ByteBuffer data){
20 20 try{
  21 + int initPos = data.position();
  22 + data.position(0);
21 23 MessageDigest md = MessageDigest.getInstance("MD5");
22 24 md.update(data);
  25 + data.position(initPos);
23 26 return md.digest();
24 27 }catch(NoSuchAlgorithmException e){
25 28 e.printStackTrace();
@@ -37,5 +40,4 @@ public synchronized static String getFor(String a, String b){
37 40 }
38 41 return null;
39 42 }
40   -
41 43 }
BIN  datastorageproject/testfile.exe
Binary file not shown

0 comments on commit e8d212b

Please sign in to comment.
Something went wrong with that request. Please try again.