forked from nanoboard/nanoboard
-
Notifications
You must be signed in to change notification settings - Fork 2
/
NBPack.cs
1446 lines (1293 loc) · 62.6 KB
/
NBPack.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
using System;
using System.IO;
using Newtonsoft.Json;
using nboard;
using System.Collections.Generic;
using System.Linq;
using System.Drawing;
using System.Net;
using System.Text;
using System.Threading;
using System.Text.RegularExpressions; //regex to check dataURL
using System.Drawing.Imaging; //ImageFormat.Png to save bmp as PNG...
using NDB;
namespace nbpack
{
public class NBPackMain
{
public static bool allowReput = false; //true if need to allow reput posts (deleted forever or damaged/corrupted posts will re-added in the database).
public static bool bypassValidation = !captcha.Captcha.captcha_found;
public static bool enable_JSON_output = false; //if need to enable output in .json files when Autoparse_folder or unpack_JSON is working.
public static NDB.PostDb PostDatabase = new PostDb(false); //to get posts from NDB.PostDb and PutPost there. See PostDb.cs
private static object _lock = new object();
/*
The following dictionary extracted from .json-file. This file need to fix posts, and this have the following json-format:
{
//"post_hash_to_replace": {"reply_to", "new_correct_post_message_to_replace_on_this"},...,{the_same_data_for_new_post}
"post1_hash_to_replace": [
"new_post1_hash",
"new_post1_message_escaped_by_@"
],
"post2_hash_to_replace": [
"new_post2_hash",
"new_post2_message_escaped_by_@"
],
"post2_hash_to_replace": [
"new_post3_hash",
"new_post3_message_escaped_by_@"
],
//invalid posts, which was been deleted forever, or posts with wipe.
"post4_hash_to_replace": ["deleted", "forever"],
"post5_hash_to_replace": ["deleted", "forever"],
"post6_hash_to_replace": ["deleted", "forever"]
//etc...
}
Post message, can contains JSON-characters or special characters,
So, for escape post-message, can be used JavaScript functions:
encodeURIComponent(unescaped_string)
decodeURIComponent(escaped_string)
Or CSharp functions:
System.Uri.EscapeDataString(string unescaped_string)
System.Uri.UnescapeDataString(string escaped_string)
*/
public static string fix_posts_json_file = "fix_posts.json"; //file with this json
public static Dictionary<string, List<string>> fix_posts = new Dictionary<string, List<string>>(); //dictionary to fix_posts from this json-file.
public static void Main(string[] args)
{
if (args.Contains("bypassValidation")){bypassValidation = true;}
if (args.Contains("enable_JSON_output")){enable_JSON_output = true;}
if (File.Exists(@fix_posts_json_file)){
string json = File.ReadAllText(@fix_posts_json_file);
fix_posts = JsonConvert.DeserializeObject<Dictionary<string, List<string>>>(json);
foreach (KeyValuePair<string, List<string>> key_value in fix_posts)
{
//if(key_value.Value[0] == "deleted" && key_value.Value[1] == "forever"){continue;}
// (key_value.Value).Add(HashCalculator.Calculate(key_value.Value[0] + key_value.Value[1]));
(key_value.Value).Add(nboard.HashCalculator.Calculate(key_value.Value[0] + key_value.Value[1]));
// (key_value.Value).Add(NDB.HashCalculator.Calculate(key_value.Value[0] + key_value.Value[1]));
}
}
if (!Directory.Exists("upload"))
Directory.CreateDirectory("upload");
if (!Directory.Exists("download"))
Directory.CreateDirectory("download");
if (!Directory.Exists("containers"))
Directory.CreateDirectory("containers");
if (
args.Length <= 2 //if "nanodb.exe NBPack -x" or "NBPack -x" (not full args)
|| //or
(
new string[] { "-h", "--h", "-help", "--help"} //if any value from this list
.Any(s => s == args[0]) //in args
)
)
{ //show usage, then
Console.WriteLine(@"Simple tool that can pack/unpack png containers of 1.x nanoboard format,
also can talk to 2.0 server to:
1) feed posts from containers (download folder) to 2.0
2) extract posts from 2.0 to form a container
This is the main intent of this tool - to help 2.0 to use transport
compatible with 1.x nanoboard PNG containers format.
Example usages:
nbpack
nbpack -help
nbpack -h
(show this usage).
nbpack -g http://127.0.0.1:7346 nano queue_image_params
(creates PNG container using random picture
from ""containers"" folder and puts result into ""upload"" folder)
nbpack -o http://127.0.0.1:7346 nano ""container.png"" True
(Open file ""container.png"", extract posts into database
and save this or delete - True/False)
nbpack -a http://127.0.0.1:7346 nano
(for each picture from ""download"" folder,
tries to unpack it as a container,
then sends it's posts to the 2.0 server and deletes it)
nbpack -a_folder ""FolderWithContainers"" nano
(make autoparse for each container
in ""FolderWithContainers"" with key ""nano"")
Other usages (may be useful if you're developing your own client):
nbpack -v ""posts.json"" ""output.json""
(rewrite hashes from ""posts.json"" to ""output.json"")
nbpack -v_base64 ""posts.json"" ""output.json""
(rewrite hashes from ""posts.json""
to ""output.json"" and encode messages to base64)
nbpack -p posts.json template.png crypto_key output.png
(pack ""posts.json"" from ""template.png""
with key ""crypto_key"" to ""output.png"")
nbpack -u container.png crypto_key output.json
(unpack container from ""container.png""
with key ""crypto_key"" to ""output.json"")
nbpack -pf ""file.ext"" ""PNG.png"" nano ""container.png""
(pack-file ""file.ext"" from ""PNG.png""
with key ""nano"" into ""container.png"")
nbpack -uf ""container.png"" nano ""file.ext""
(unpack-file from ""container.png""
with key ""nano"" to ""file.ext"")
nbpack -gen args
(generate png with args)
"
/*
+@" nbpack -generate_captcha args
(generate a new captcha-pack-file, with args.)
There is already exist an official captcha-pack file - "captcha.nbc".
With new captcha full-validation of old containers is not working.
"
*/
+@"
""enable_JSON_output"" and ""bypassValidation""
are optional and additional parameters.
By default,
""enable_JSON_output""=false,
""bypassValidation"" Frue (if captcha-file found) or False.
Sample JSON (note that message contains utf-8 BYTES converted to base64 string)
{ ""posts"" : [ { ""hash"" : "".."", ""replyTo"" : "".."", ""message"" : ""base64"" }, .. ] }
"
);
Console.ReadKey(); //do not close the window, if program was been runned by double-click, and wait to press any key.
return;
}
string address = "";
string key = "";
switch (args[0])
{
// Generate container from database
case "-g":
string queue_image_params = "";
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}else{
address = args[1];
key = args[2];
}
if (args.Length == 4){
queue_image_params = args[3];
}
Console.WriteLine("Start create container from database.");
Create(address, key, queue_image_params);
break;
// Open specified container and add posts to database
case "-o": //open container
string container = "";
bool save_files = true;
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
else if (args.Length == 3) //-o file.png key
{
container = args[1];
key = args[2];
}
else if (args.Length == 4) //-o file.png key save_files(True/False)
{
container = args[1];
key = args[2];
save_files = bool.Parse(args[3]);
}
else if (args.Length == 5) //-o address key file.png save_files(True/False)
{
address = args[1];
container = args[2];
key = args[3];
save_files = bool.Parse(args[4]);
}
Console.WriteLine("Start open container "+container+" with key "+key+". save_files ="+save_files);
ParseFile(address, key, container, save_files);
break;
// AutoParse download-folder into database
case "-a":
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
address = args[1];
key = args[2];
Console.WriteLine("Start Autoparse container from \"download\"-folder into database.");
AutoParse(address, key);
break;
// Autoparse all containers in the specified folder, and save results in the same folder.
case "-a_folder":
string folder_with_containers = "";
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
folder_with_containers = args[1];
key = args[2];
Console.WriteLine("Start autoparse containers in folder: "+folder_with_containers+" with key: "+key);
AutoParse_folder(folder_with_containers, key); //+ optionally enable_JSON_output and bypassValidation
break;
// Validate posts.
case "-v": // validate
string raw_posts = "";
string validated_posts = "";
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
raw_posts = args[1];
validated_posts = args[2];
Console.WriteLine("Start rewrite hashes from json-file with posts: "+raw_posts+", to json-file with posts: "+validated_posts);
Validate(raw_posts, validated_posts);
break;
// Validate raw-posts - add hashes into the JSON file with replies, and encode messages to base64
// to make result acceptable with enabled bypassValidation (without solved POW and Captcha)
case "-v_base64": //validate (base64)
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
raw_posts = args[1];
validated_posts = args[2];
Console.WriteLine("Add hashes to the posts, and encode raw messages, if still not base64-encoded...");
Validate_base64(raw_posts, validated_posts); //filepath of file with posts without hashes, filepath of file with posts with hashes.
break;
// pack "validated_posts" from "json-file" to container from image "templatePath", encrypt it with "key", and save in "outputPath"
case "-p": //pack
//Pack(string validated_JSON_postsPath, string templatePath, string key, string outputPath)
string validated_JSON_postsPath = "";
string templatePath = "";
if (args.Length < 5)
{
Console.WriteLine("Insufficient parameters count");
return;
}
validated_JSON_postsPath = args[1];
templatePath = args[2];
key = args[3];
container = args[4];
Console.WriteLine("Start pack validated_posts from json-file: "+validated_JSON_postsPath+", to image from \""+
templatePath+"\", encrypt it with key "+key+", and save in "+container);
Pack(validated_JSON_postsPath, templatePath, key, container);
break;
// unpack posts from container to HTML-file, like in old unpack.exe, and optionally - into JSON-file (if enable_JSON_output) is specified.
// and base64 encoded messages and with solved capthcha (if bypassValidation not specified)
case "-u": // unpack
string outputPath = "";
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
container = args[1];
key = args[2];
outputPath = args[3];
//In some case, maybe, need to run Unpack with two parameters, from another method, because this have type NDB.Post[], not void
Console.WriteLine("Start unpack container: "+container+", decrypt posts with key "+key+", and save in "+outputPath);
Unpack(container, key, outputPath); //void Unpack posts from container
break;
// pack file into PNG, and encrypt it by specified key
case "-pf": //pack file
string filepath_to_pack = "";
if (args.Length < 4)
{
Console.WriteLine("Insufficient parameters count");
return;
}
filepath_to_pack = args[1];
templatePath = args[2];
key = args[3];
container = args[4];
Console.WriteLine("Start pack file "+filepath_to_pack+"from png "+templatePath+" with key "+key+" in container "+container);
PackFile(filepath_to_pack, templatePath, key, container); //void (string filepath_to_pack, string templatePath, string key, string outputPath)
break;
// unpack file from PNG-container, and decrypt the data by specified key
case "-uf": // unpack file
string outputPath_folder = "";
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}else if(args.Length==3){
//run Unpack with two parameters, from previous function, because this have type NDB.Post[], not void
return;
}
container = args[1];
key = args[2];
outputPath_folder = args[3];
Console.WriteLine("Start unpack file from container:"+container+", decrypt it with key "+key+", and save in the folder: "+outputPath_folder);
UnpackFile(container, key, outputPath_folder); //void (string containerPath, string key, string outputPath)
break;
// Generate PNG image in specified folder
case "-gen": //generate PNG
if (args.Length < 3)
{
Console.WriteLine("Insufficient parameters count");
return;
}
Console.WriteLine("Start generate new random PNG...");
generate_PNG(args); //args[1] = (string) folder , args[2] = (string) param1;param2;param3
break;
// Generate Captcha-Pack-file.nbc
case "-generate_captcha": //generate captcha
args = args.Skip(1).ToArray();
Console.WriteLine("Start generate new captcha-pack-file...");
CaptchaPack_Generator.Program.Main(args); //generate_captcha_pack_file();
break;
}
Console.WriteLine("Done!");
//Console.ReadKey(); //temporary enable, to test, and don't close window, but need to disable, to run this by server-request.
return;
}
//int.Parse and Int32.Parse working bad for me. See issue: https://github.com/nanoboard/nanoboard/issues/5
//So this function was been writed, to make this code more independent...
public static int parse_number(string string_number)//this function return (int)number from (string)"number". Negative numbers supporting too.
{ if(string_number=="" || string_number == null){Console.WriteLine("NbPack.cs. parse_number. string_number is empty or null: (string_number == \"\"): "+string_number+", (string_number == null): "+(string_number == null)); return 0;}
string test = (new Regex(@"\D")).Replace(string_number, "");
int test_length = test.Length;
int number = 0;
for(int i = ((char)test[0]=='-')?1:0; i < test_length; i++){
number += ((int)Char.GetNumericValue((char)test[i])*(int)Math.Pow(10,test_length-i-1));
}
number = ((char)test[0]=='-'?(0-number):(number));
return number;
}
//check is base64 encoded.
public static bool IsBase64Encoded(string str)
{
try
{
// If no exception is caught, then it is possibly a base64 encoded string
byte[] data = Convert.FromBase64String(str);
// The part that checks if the string was properly padded to the
// correct length was borrowed from d@anish's solution
return (str.Replace(" ","").Length % 4 == 0);
}
catch
{
// If exception is caught, then it is not a base64 encoded string
return false;
}
}
private static bool ByteCountUnder(List<NDB.Post> posts, int limit)
{
// Console.WriteLine("NBPack.cs. ByteCountUnder: int limit: "+limit); //maybe this change limit in MaxConsecZeros
int byteCount = 0;
foreach (var p in posts)
{
byteCount += Convert.FromBase64String(p.message).Length + 32;
if (byteCount > limit) return false;
}
return true;
}
private static int ByteCount(NDB.Post p)
{
return Convert.FromBase64String(p.message).Length + 32;
}
public static string result_create = ""; //here will be saved result of working Create method.
/*
Takes 50 or less last posts (up to 150000 bytes max total),
adds 50 or less random posts (up to 150000 bytes max total),
random is shifted towards latest posts.
*/
private static void Create(string address, string key, string queue_image_params = "")
{
int from_last_posts = -1;
int start_post_number = -1;
string dataURL = "";
string queue = ""; //string with hashes of posts, separated with comma.
int from_queue = 0;
int random_posts = -1;
int max_bytelength = 150000;
//Console.WriteLine("queue_image_params {0}", queue_image_params);
string[] splitted = queue_image_params.Split('\n');
for(int i=0;i<splitted.Length;i++){
//Console.WriteLine("splitted[i] = {0}", splitted[i]);
if(splitted[i]==""){
//Console.WriteLine("Splitted[i] == \"\". Continue..."); //if empty string - continue. String is empty if '\n' at last of post-content.
continue;
}
if(
splitted[i].IndexOf("data:") != -1
&& splitted[i].IndexOf("base64,")!= -1
&& nbpack.NBPackMain.IsBase64Encoded(splitted[i].Split(',')[1])
){
dataURL = splitted[i];
}else if(splitted[i]=="No_dataURL_specified_for_source_image."){
//Console.WriteLine("DataURL not specified...");
dataURL = splitted[i];
}else if(splitted[i].StartsWith("max_bytelength=")){
//max_bytelength=
int parsed_value = nbpack.NBPackMain.parse_number(splitted[i].Substring(14));
max_bytelength = (parsed_value!=0) ? parsed_value : max_bytelength;
// Console.WriteLine(
// "Set max_bytelength = "+max_bytelength+", parsed_value = "+parsed_value+","
// +"\n(splitted[i]==\"\") = "+(splitted[i]=="")+", splitted[i] = "+splitted[i]
// );
}else if(splitted[i].Length>=32){
//Console.WriteLine("splitted.Length = 1, splitted[i] = {0}, maybe this is queue in GET-query...", splitted[i]);
queue = splitted[i];
}else if(splitted[i].Length<32){
//Console.WriteLine("i = {0}, splitted[i] = {1}, splitted[i].Length = {2} < 32", i, splitted[i], splitted[i].Length);
if(i==0){
//Console.WriteLine("splitted[i], i==0, this is first value with length lesser than 32, maybe number = from_last_posts{0}", splitted[i]);
if(splitted[i].IndexOf('-')!=-1){ //if range specified
string [] from_to = splitted[i].Split('-'); //split this
if(from_to.Length==2){ //if array length ==2 - set two numbers
start_post_number = nbpack.NBPackMain.parse_number(from_to[0]); //start post
from_last_posts = nbpack.NBPackMain.parse_number(from_to[1]); //posts count to add
}else{ //else
Console.WriteLine( "Negative numbers not supporting in parameters! \n"+ //maybe negative number was been sent
"First parameter contains - and not a range: "+splitted[i]
);
return; //error
}
}
else{from_last_posts = nbpack.NBPackMain.parse_number(splitted[i]);}
}else if (i==1){
//Console.WriteLine("splitted[i], i>0, this is not first value with length lesser than 32, maybe number = from_queue{0}", splitted[i]);
from_queue = nbpack.NBPackMain.parse_number(splitted[i]);
}else/*if (i==2)*/{
random_posts = nbpack.NBPackMain.parse_number(splitted[i]);
}
}
//Console.WriteLine("End of iteration...");
}
//Console.WriteLine("End of cycle...");
if(dataURL==""){//if string is null, this is empty string
//Console.WriteLine("dataURL still empty...");
dataURL="No_dataURL_specified_for_source_image.";
}
if(from_queue==0){from_queue = queue.Split(',').Length;}
//Console.WriteLine("Queue {0}", queue);
//Console.WriteLine(
// "from_last_posts = {0}\n"
// +"dataURL = {1}\n"
// +"queue = {2}\n"
// +"from_queue = {3}\n"
// ,
// from_last_posts
// ,dataURL.Substring(0, ( (dataURL.Length>40) ?40 :dataURL.Length ) )
// ,queue
// ,from_queue
//);
List<NDB.Post> list = new List<NDB.Post>(); //define empty list
var count = 0; //define count variable
int take = -1; //number of posts to taking (default value -1)
if( //if queue hashes not defined here
queue==null //and this is default value, null
|| queue == "" //or if this is defined, but empty string
){ //pack last from_last_posts
//Console.WriteLine("Queue is empty. Pack last {0} posts...", from_last_posts); //Show message about this
count = PostDatabase.GetPresentCount(); //get posts count (total posts)
if(from_last_posts!=-1 && from_last_posts!=0){
Console.WriteLine("Pack last {0} posts...", from_last_posts);
take = from_last_posts; //from_last_posts posts to taking
}else if(from_last_posts==0){
take = 0;
}else{
Console.WriteLine( "Number of last posts to pack - not specified!\n"+
"Pack last 50 posts, by default..."
);
take = 50; //from_last_posts posts to taking
}
// Console.WriteLine("start_post_number = "+start_post_number+", (count - take) = "+(count - take)+
// "\n( (start_post_number!=-1) ? start_post_number : (count - take) )"+( (start_post_number!=-1) ? start_post_number : (count - take) )+
// "\n(Math.Max( ( (start_post_number!=-1) ? start_post_number : (count - take) ), 0))"+Math.Max( ( (start_post_number!=-1) ? start_post_number : (count - take) ), 0)+
// "\ntake = "+take
// );
var last50s = PostDatabase.RangePresent(Math.Max( ( (start_post_number!=-1) ? start_post_number : (count - take) ), 0), take); //take last 50 posts
list = last50s.ToList(); //push this to list.
}else{ //if queue hashes defined
var posts = PostDatabase.GetPosts(queue, from_queue); //get array with posts
//count = posts.Length; //get post count in queue
count = PostDatabase.GetPresentCount(); //get posts count in database (total posts)
//Console.WriteLine("Queue accepted. Pack {0} posts from {1} posts in queue...", from_queue, count); //show message about pack posts from queue
list = posts.ToList(); //push posts to list
}
//select random container file
var files = Directory.GetFiles("containers", "*.png").ToList();
files.AddRange(Directory.GetFiles("containers", "*.jpg"));
files.AddRange(Directory.GetFiles("containers", "*.jpeg"));
string add_notif = "";
if (files.Count == 0)
{ add_notif = ((address == "CreatePNG_on_lite_server")?address+". ":"")+"Your containers dir is empty! Add container(s)"; result_create += add_notif+"|||"; //NServer.DbApiHandler.notifications_with_filename += add_notif+"|||"; //NServer.NotificationHandler.Instance.Messages.Enqueue("Your containers dir is empty! Add container(s)"); //add notif from response.
// NServer.NotificationHandler.Instance.Messages.Enqueue("Your containers dir is empty! Add container(s)");
return;
}
var r = new Random();
var file = files[r.Next(files.Count)];
var name = ( (address == "CreatePNG_on_lite_server") ? "download/created/" : "upload/" )+ Guid.NewGuid().ToString() + ".png";
//begin calculate capacity
Image bmp = null;
if(dataURL=="No_dataURL_specified_for_source_image."){
bmp = Bitmap.FromFile(file);
}
else if(
dataURL.IndexOf("data:") != -1
&& dataURL.IndexOf("base64,")!= -1
&& nbpack.NBPackMain.IsBase64Encoded(dataURL.Split(',')[1])
){
Console.WriteLine("Image uploaded and dataURL found. Create bitmap from dataURL.");
//create bitmap from dataURL, and save this as PNG-file to Upload folder.
var base64Data = Regex.Match(dataURL, @"data:image/(?<type>.+?),(?<data>.+)").Groups["data"].Value;
var binData = Convert.FromBase64String(base64Data);
using (var stream = new MemoryStream(binData))
{
bmp = new Bitmap(stream); //create image from dataURL
}
}else{
Console.WriteLine("NBPack.cs - Create method: No DataURL found.");
}
var capacity = ((bmp.Width * bmp.Height * 3) >> 3) - 4; //each bit in RGB subpixel byte - 32. Result = total bytes can be packed.
capacity = (capacity>max_bytelength) ? capacity : max_bytelength; //if capacity < 150000 bytes, resize image.
//Console.WriteLine("bmp.Width = "+bmp.Width+", bmp.Height = "+bmp.Height+", capacity = "+capacity+", max_bytelength = "+max_bytelength);
bmp.Dispose();
//end calculate capacity
int deleted_post_number = 0;
while (!ByteCountUnder(list, capacity)) //remove old static limit, and check bytecount - up to capacity, to do not do resize image.
{
Console.WriteLine(
"removed_post_hash: "+list[0].hash
+"! Deleted post nubmer = "+(++deleted_post_number)
);
list.RemoveAt(0);
}
if(random_posts==-1){
Console.WriteLine( "Number of random posts to pack not specified!"+
"Pack random 50 posts, by default..."
);
random_posts = 50;
}else if(random_posts!=0){
Console.WriteLine( "Add {0} random posts to container", random_posts);
}else{
//don't show nothing if 0 posts added.
}
//add random posts
int rbytes = 0;
for (int i = 0; i < random_posts; i++)
{
int index = (int)Math.Min(Math.Pow(r.NextDouble(), 0.3) * count, count - 1);
var p = PostDatabase.RangePresent(index, 1)[0];
var bc = ByteCount(p);
if (rbytes + bc > capacity){
Console.WriteLine("{0} posts selected from last {1}, according of bytelimit", i, random_posts);
break;
}
rbytes += bc;
if(!list.Contains(p)){list.Add(p);}
}
//Shuffle all elements in list - to anonymize container creator.
list = list.OrderBy(a => Guid.NewGuid()).ToList();
//pack from file or from specified dataURL
if(dataURL=="No_dataURL_specified_for_source_image."){
Pack(list.ToArray(), file, key, name, address);
}else{
Pack(list.ToArray(), dataURL, key, name, address);
}
List<string> packed_posts_hashes = new List<string>();
foreach (var p in list)
{
packed_posts_hashes.Add(p.hash);
}string add_text = ((address == "CreatePNG_on_lite_server")?address+". ":"");
//if(add_text!=""){NServer.DbApiHandler.datetime_of_generated_images[name] = DateTime.Now; NServer.DbApiHandler.timer_to_delete.Start();} //if this is request on lite-server
add_notif = add_text+"Saved PNG to /" + name; /* NServer.NotificationHandler.Instance.Messages.Enqueue(add_notif+"|||"); //notifs will be added, from splitted response, by using pushNotification()*/ result_create += add_notif+"|||"; //NServer.DbApiHandler.notifications_with_filename += add_notif+"|||";
add_notif = add_text+"Hashes of posts, packed into "+name+": " + JsonConvert.SerializeObject(packed_posts_hashes); /*NServer.NotificationHandler.Instance.Messages.Enqueue(add_notif+"|||"); //do not add notif */ result_create += add_notif+"|||"; //NServer.DbApiHandler.notifications_with_filename += add_notif+"|||";
return; //return HttpResponse for request on /api/png-create, in DbApiHandler.cs. Method PngCreate.
}
public static string ParseFile_result = "";
public static void ParseFile(string address, string key, string filename, bool save_files=false) //filename - is pathway for container file.
{
lock(_lock){//Console.WriteLine("ParseFile: save_files = "+save_files);
var posts = Unpack(filename, key); //here catch for some files.
GC.Collect();
try
{
foreach (var p in posts)
{
bool added = PostDatabase.PutPost(p, allowReput, bypassValidation);
if (added)
{
// NServer.NotificationHandler.Instance.
// Messages.Enqueue( "[b][g]Extracted post:[/g][/b] "
// + Encoding.UTF8.GetString(Convert.FromBase64String(p.message)));
ParseFile_result += "[b][g]Extracted post:[/g][/b] "
+ Encoding.UTF8.GetString(Convert.FromBase64String(p.message));
}
}
}
catch (Exception e)
{
Console.WriteLine("NBPack.cs: ParseFile. Try add posts - catch: "+e.Message);
}
// while(true){ //wanted to use infinite cycle with repeat this... fail, because validate();
if(!save_files){
try
{
File.Delete(filename);
//Console.WriteLine("Delete: "+filename);
// break;
}
catch(Exception e)
{
Console.WriteLine("NBPack.cs: ParseFile(pathway). Try to delete file - catch: "+e.Message); //from here is error message for some files.
//System.Threading.Thread.Sleep(10);
}
}}
return;
}
public static void ParseFile(string address, string key, Image container) //here Image from RAM
{
lock(_lock){var posts = Unpack(container, key); //here can be catch for some Images
GC.Collect();
try
{
int posts_added = 0;
foreach (var p in posts)
{
//if(bypassValidation){ Console.WriteLine("Captcha file not found. bypassValidation = "+bypassValidation+"now..."); }
bool added = false;
try{
added = PostDatabase.PutPost(p, allowReput, bypassValidation); //don't allow reput post again.
}catch(Exception ex){
Console.WriteLine("Try to PutPost: "+ex);
}
if (added)
{
/*
try{
NServer.NotificationHandler.Instance.
Messages.Enqueue(
//"[b][g]Extracted post:[/g][/b] "
//+ Encoding.UTF8.GetString(Convert.FromBase64String(p.message)) //display post
"[b][g]Extracted post:[/g][/b] "
+ p.hash //only hash
);
}
catch(Exception ex){ //here sometimes catch srcIndex for fast collect
Console.WriteLine("Try to add notification: "+ex + "\nstring.Length" + Encoding.UTF8.GetString(Convert.FromBase64String(p.message)).Length);
}
*/
posts_added++;
}
/*
//or don't display posts and just display added posts:
try{
NServer.NotificationHandler.Instance.
Messages.Enqueue(
"[b][g]Contains posts:[/g][/b] " //Display contains posts
+ posts.Length + //posts in container
". Added: "
// + posts_added //added posts
// + "posts."
//or this notification
+
(
(posts_added==posts.Length) //if all posts added
? "All" //display "All"
: posts_added + //or display added posts
", Not added: " + //and not added
(posts.Length - posts_added) //value, then.
+(
( ( posts.Length - posts_added ) != 0 )
? " (maybe already exist)"
: ""
)
)
);
}
catch(Exception ex){
Console.WriteLine("Try to add notification: "+ex); //sometimes error: srcIndex
}
*/
}Console.WriteLine(posts_added+" new posts added to database");//, bypassValidation = "+bypassValidation);
}
catch (Exception e)
{
Console.WriteLine("NBPack.cs: ParseFile(Image RAM) - try to add posts: "+e.Message); //here catch "OutOfMemoryException" for notifs.
}}
return;
}
private static void AutoParse(string address, string key)
{
lock(_lock){var files = Directory.GetFiles("download");
foreach (var f in files)
{
var posts = Unpack(f, key);
GC.Collect();
try
{
foreach (var p in posts)
{
PostDatabase.PutPost(p, allowReput, bypassValidation);
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
try
{
File.Delete(f);
}
catch(Exception e)
{
Console.WriteLine(e.Message);
}
}}
return;
}
//parse all files from the folder "folder_with_containers" with key, or and with specified bypassValidation,
private static void AutoParse_folder(string folder_with_containers, string key)
{
Console.WriteLine("AutoParse. folder_with_containers: "+folder_with_containers);
lock(_lock){
string[] files = Directory.GetFiles(folder_with_containers);
int file_number = 0;
foreach (string f in files)
{
if(f.EndsWith(".png")){
Console.WriteLine("Try unpack file "+file_number+" with filename: "+f);
Unpack(f, key, f.Replace(".png", ".html")); //two last params "enable_JSON_output" and bypassValidation was defined globally
file_number += 1;
}
}
}
return;
}
//Add hashes to the posts, in array NDB.Post[]
private static void Validate(NDB.Post[] posts)
{
foreach (var p in posts) //for each post in array NDB.Post[] posts
{
if(!nbpack.NBPackMain.IsBase64Encoded(p.message)){ //if post-message not base64-encoded
p.message = Convert.ToBase64String(Encoding.UTF8.GetBytes((@p.message).Replace("\r\n", "\n"))); //encode it as base64, and save
}
p.hash = nboard.HashCalculator.Calculate(p.replyto + Encoding.UTF8.GetString(Convert.FromBase64String(p.message))); //Then, calculate hash and add this p.hash
}
return; //return, after this all.
}
//Add hashes to the posts in json-fle in postsPath, and save this in json-file in outputPath (simple validation in nanoboard 1.0)
private static void Validate(string postsPath, string outputPath)
{
var json = File.ReadAllText(postsPath); //read all JSON as text from postsPath-file
var posts = JsonConvert.DeserializeObject<NDB.Post[]>(json); //Deserialize this in array NDB.Post[] posts
Validate(posts); //add hashes for the posts in this array
var result = JsonConvert.SerializeObject(posts, Formatting.Indented); //save this as JSON with Formatting.Indented
File.WriteAllText(outputPath, result); //Write this JSON as text in outputPath
return; //and return, after all.
}
//just add hashes for posts[i], in posts[i].hash, and base64 encoded message in posts[i].message, if this message was not been encoded.
private static void Validate_base64(nbpack.Post[] posts)
{
for(var i = 0; i<posts.Length; i++)
{
byte[] message_as_bytes = null;
try{
message_as_bytes =
Encoding.UTF8.GetBytes(
posts[i].message //take message for i-th post from posts
.FromB64() //try to decode this from base64, or unescape, to get raw-message
.Replace("\r\n", "\n") //and replace "CRLF" to "LF" (if Windows EOL)
.Replace("\r", "\n") //or/and "CR" to "LF", then (if Mac EOL, or mixed EOL)
) //and decode this to bytes, as UTF8-encoded text.
;
}
catch(Exception ex){
Console.WriteLine("NBPack.cs, Validate_base64, ex: "+ex); //show this, and leave message_bytes as null.
}
posts[i].message = Convert.ToBase64String(message_as_bytes);
posts[i].hash = nboard.HashCalculator.Calculate(posts[i].replyto + Encoding.UTF8.GetString(Convert.FromBase64String(posts[i].message)));
}
return;
}
//Just add hashes of the posts without captcha and pow, to be this accepted with enabled bypassValidation, and write this posts in outputPath, as JSON.
private static void Validate_base64(string raw_postsPath, string validated_outputPath)
{
var json = File.ReadAllText(raw_postsPath);
bool _bypassValidation = bypassValidation;
if(bypassValidation == false && ( !json.Contains("[pow=") && !json.Contains("[sign="))){
Console.WriteLine(@"Noone post not contains ""pow"" and ""sign""-tags. Try to using bypassValidation for this posts.");
bypassValidation = true;
}
try{
var posts = JsonConvert.DeserializeObject<nbpack.Post[]>(json);
Validate_base64(posts);
posts = NDB.PostsValidator.Validate(posts, bypassValidation); //validate captcha or rewrite hashes
Console.WriteLine("validated posts.Length: "+posts.Length);
var result = JsonConvert.SerializeObject(posts, Formatting.Indented);
File.WriteAllText(validated_outputPath, result);
}
catch(Exception ex){
Console.WriteLine("Validate_base64: "+ex);
}
bypassValidation = _bypassValidation;
return;
}
//Pack NDB.post[] posts, from DataBase, when Create PNG is runned,
//pack this array with posts in the container, from specified templatePatch.png (this value can be a dataURL),
//encrypt packed data by key, and save in outputPath.png
//optional parameter "address", using to append string, when "CreatePNG" is runned on lite-server.
private static void Pack(NDB.Post[] posts, string templatePath, string key, string outputPath, string address = "")
{
var @set = new HashSet<string>();
Validate(posts); //NDB.Post[] posts
var nposts = new List<NanoPost>();
string add_notif = ((address == "CreatePNG_on_lite_server")?address+". ":"")+"Showing posts that will go to the container:";
//NServer.NotificationHandler.Instance.Messages.Enqueue(add_notif); //do not add notif, and add this from response.
//NServer.DbApiHandler.notifications_with_filename += add_notif + "|||";
result_create += add_notif + "|||";
foreach (var p in posts)
{
var mess = Encoding.UTF8.GetString(Convert.FromBase64String(p.message));
var hash = p.hash;
if (!@set.Contains(hash))
{
@set.Add(hash); /*add_notif = ((address == "CreatePNG_on_lite_server")?address+". ":"")+mess; //do not define this, because nothing to do with this.
NServer.NotificationHandler.Instance.Messages.Enqueue(add_notif); //do not add notif, and add this from response*/ /* NServer.DbApiHandler.notifications_with_filename += add_notif + "|||";// don't add posts-content - to returned string */
}
nposts.Add(new NanoPost(p.replyto + mess));
}
var packed = NanoPostPackUtil.Pack(nposts.ToArray());
var encrypted = ByteEncryptionUtil.EncryptSalsa20(packed, key);
Image bmp;
if(
templatePath.IndexOf("data:") != -1
&& templatePath.IndexOf("base64,")!= -1
&& nbpack.NBPackMain.IsBase64Encoded(templatePath.Split(',')[1])
){
Console.WriteLine("Image uploaded and dataURL found. Create bitmap from dataURL.");
//create bitmap from dataURL, and save this as PNG-file to Upload folder.
var base64Data = Regex.Match(templatePath, @"data:image/(?<type>.+?),(?<data>.+)").Groups["data"].Value;
var binData = Convert.FromBase64String(base64Data);
using (var stream = new MemoryStream(binData))
{
bmp = new Bitmap(stream); //create image from dataURL
//save this image as PNG-file to the folder "upload"
//Console.WriteLine(bmp);
//bmp.Save("upload/" + Guid.NewGuid().ToString() + ".png", ImageFormat.Png);
//bmp.Dispose();
//Console.WriteLine("saved to \"upload\"");
}
//working...
}else{
//Console.WriteLine("DataURL not found. Create bitmap from templatePath = {0}", templatePath);
bmp = Bitmap.FromFile(templatePath);
}
var capacity = (bmp.Width * bmp.Height * 3) / 8 - 32;
if (encrypted.Length > capacity)
{
float scale = (encrypted.Length / (float)capacity);
Console.WriteLine("Warning: scaling image to increase capacity: " + scale.ToString("n2") + "x");
scale = (float)Math.Sqrt(scale);
bmp = new Bitmap(bmp, (int) (bmp.Width * scale + 1), (int) (bmp.Height * scale + 1));
}
new PngStegoUtil().HideBytesInPng(bmp, outputPath, encrypted);
return;
}
//Pack posts from json in container outputPath, which is created from PNG in templatePath, and encrypt data by key
private static void Pack(string validated_JSON_postsPath, string templatePath, string key, string outputPath)
{
try{
var json = File.ReadAllText(validated_JSON_postsPath);
var posts = JsonConvert.DeserializeObject<NDB.Post[]>(json); //NDB.Post[] posts
// var posts = JsonConvert.DeserializeObject<nbpack.Post[]>(json); //or nbpack.Post[] posts
Console.WriteLine("Pack: posts accepted: "+posts.Length);
//Add hashes, if there is no any hashes.
Validate(posts); //NDB.Post[] posts