Skip to content

Commit

Permalink
Minor improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
waybackarchiver committed Aug 25, 2021
1 parent 1dc3a95 commit d5bead8
Show file tree
Hide file tree
Showing 11 changed files with 131 additions and 71 deletions.
15 changes: 6 additions & 9 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ module github.com/wabarc/wayback
go 1.16

require (
github.com/PuerkitoBio/goquery v1.7.1 // indirect
github.com/btcsuite/btcd v0.22.0-beta // indirect
github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/chromedp/cdproto v0.0.0-20210808225517-c36c1bd4c35e // indirect
github.com/chromedp/cdproto v0.0.0-20210823203301-2c0adcc9edc4 // indirect
github.com/cretz/bine v0.2.0
github.com/davecgh/go-spew v1.1.1
github.com/dghubble/go-twitter v0.0.0-20201011215211-4b180d0cc78d
Expand All @@ -20,7 +19,6 @@ require (
github.com/go-shiori/go-readability v0.0.0-20210627123243-82cc33435520
github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b // indirect
github.com/google/go-github/v38 v38.1.0
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.4.2
github.com/iawia002/annie v0.0.0-20210720065824-ac5f36d99548
Expand All @@ -32,6 +30,7 @@ require (
github.com/mattn/go-mastodon v0.0.5-0.20210515144304-86627ec7d635
github.com/multiformats/go-multiaddr v0.3.3 // indirect
github.com/multiformats/go-multihash v0.0.15 // indirect
github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.30.0
Expand All @@ -48,20 +47,18 @@ require (
github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933
github.com/wabarc/screenshot v1.3.0
github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365
github.com/wabarc/screenshot v1.3.2-0.20210824153650-d47a1474a43e
github.com/wabarc/telegra.ph v0.0.0-20210822083402-82f95ce60a37
github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c
github.com/wabarc/wbipfs v0.2.0
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
go.etcd.io/bbolt v1.3.6
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
gopkg.in/tucnak/telebot.v2 v2.4.0
maunium.net/go/mautrix v0.9.19
mvdan.cc/xurls/v2 v2.3.0 // indirect
maunium.net/go/mautrix v0.9.20
)

replace github.com/go-shiori/obelisk => github.com/wabarc/obelisk v0.0.0-20210420023708-aac2bcc00a78
34 changes: 14 additions & 20 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,10 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
github.com/chromedp/cdproto v0.0.0-20210526005521-9e51b9051fd0/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20210625233425-810000e4a4fc/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20210706234513-2bc298e8be7f/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/cdproto v0.0.0-20210808225517-c36c1bd4c35e h1:uYQIvPKwL7YQh+tQC+EUXmvWosBeOYbJZKrh4NnS0UM=
github.com/chromedp/cdproto v0.0.0-20210808225517-c36c1bd4c35e/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/chromedp v0.7.3/go.mod h1:9gC521Yzgrk078Ulv6KIgG7hJ2x9aWrxMBBobTFk30A=
github.com/chromedp/cdproto v0.0.0-20210823203301-2c0adcc9edc4 h1:cD7F5LfNjC4dtb+BQe1KxxE8RsceYtS+Kyo+yEs4iUc=
github.com/chromedp/cdproto v0.0.0-20210823203301-2c0adcc9edc4/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
github.com/chromedp/chromedp v0.7.4 h1:U+0d3WbB/Oj4mDuBOI0P7S3PJEued5UZIl5AJ3QulwU=
github.com/chromedp/chromedp v0.7.4/go.mod h1:dBj+SXuQHznp6ZPwZeDDEBZKwclUwDLbZ0hjMialMYs=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
Expand Down Expand Up @@ -196,8 +193,6 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gobwas/ws v1.1.0-rc.6/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down Expand Up @@ -276,7 +271,6 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
Expand Down Expand Up @@ -467,6 +461,8 @@ github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzb
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g=
github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -597,7 +593,6 @@ github.com/wabarc/go-catbox v0.1.0 h1:/UhV9md3MJrjZtm+EToSyFjawXgPiHSExLNRqsWNis
github.com/wabarc/go-catbox v0.1.0/go.mod h1:Zjs9Y55f2WOwGWwmKSCrUuMfwh+nDktkjub9jgHq4CQ=
github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8=
github.com/wabarc/helper v0.0.0-20210407153720-1bfe98b427fe/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
github.com/wabarc/helper v0.0.0-20210613000316-cc3df6340ab5/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
github.com/wabarc/helper v0.0.0-20210614160629-1a5ba5e551eb/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
github.com/wabarc/helper v0.0.0-20210701193643-e0fe0a807cb9/go.mod h1:TuTZtoiOu984UWOf7FfX58JllKMjq7FCz701kB5W88E=
github.com/wabarc/helper v0.0.0-20210718171053-59c70d0b20c2 h1:6rMZse2rdD7N6GxHRZqHlkSptBWh/Vf9aHiFVQjlQNo=
Expand All @@ -607,7 +602,6 @@ github.com/wabarc/imgbb v1.0.0/go.mod h1:piz6QXkVfFpQbaQHx70/rPDwaICkW5Ew3lShTzb
github.com/wabarc/ipfs-pinner v1.0.1 h1:PaWBDn1X556IyiOVd143XO9PVRjbohppA+JMJQwNBHI=
github.com/wabarc/ipfs-pinner v1.0.1/go.mod h1:3EsCzUMiCsqnHr/fbHHj3r8s72CHQIm3JdiOpYyh27E=
github.com/wabarc/logger v0.0.0-20210417045349-d0d82e8e99ee/go.mod h1:4uYr9fnQaQoDk1ttTzLnSB3lZm3i/vrJwN8EZIB2YuI=
github.com/wabarc/logger v0.0.0-20210708144517-a9651f538672/go.mod h1:4uYr9fnQaQoDk1ttTzLnSB3lZm3i/vrJwN8EZIB2YuI=
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792 h1:0xD4NsQtdPCB77q4wuRn1TJOuHQ3drL0SfRd6xF4ryk=
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792/go.mod h1:2an5YHL0CegFbaDNSU3kwqYRfxFfmcwlwSi5Bk4yk5c=
github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae h1:7TLEQYM7GxeysDd6IbK5F7krZuYEipdXno9qV3eedWU=
Expand All @@ -616,11 +610,11 @@ github.com/wabarc/obelisk v0.0.0-20210420023708-aac2bcc00a78 h1:iWcCLB7Ru5lurvN0
github.com/wabarc/obelisk v0.0.0-20210420023708-aac2bcc00a78/go.mod h1:OlGUo0utWqPMcej0HirsoK2+65Je/ZPFNwasF/O3KYM=
github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933 h1:KZHhUK+ZRhe4OxQ0+/H40jsS8NCCGJlY1jO7mzOdYyY=
github.com/wabarc/playback v0.0.0-20210718054702-cab6c6004933/go.mod h1:or8QaM2wUqIX45GfmLbky8AZKgOQm78VueJpqY2TwXY=
github.com/wabarc/screenshot v1.2.1-0.20210708225510-eb68213a95f1/go.mod h1:lT2loRSha0/gQni5A5JAv807xaEOztrIzCPuTQIs9zs=
github.com/wabarc/screenshot v1.3.0 h1:5gTP6lWHn77qnRM47Ap3vx/9/HV3fHPV5jX0an9Nilw=
github.com/wabarc/screenshot v1.3.0/go.mod h1:YGY7i/ZyMqJM3aYBjpsp+j7m68W2hW1d+sDAA61p90g=
github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365 h1:AZeSe84nDyD5bYNpkCVVaOxdvaHIOj5tjVJPqcjjFLI=
github.com/wabarc/telegra.ph v0.0.0-20210718173528-f72fc9cdc365/go.mod h1:e2nxaKoyr34W/jZefnrTn4HGXKdg0HBygLvIDJKFAiw=
github.com/wabarc/screenshot v1.3.1/go.mod h1:ei8rqXW5mdztkqcsb81YUVwBZFROgyjAQJrEKEiMWfY=
github.com/wabarc/screenshot v1.3.2-0.20210824153650-d47a1474a43e h1:T8gDPsE6STy7nDgIvyaP4zXGvLv7CmfQm4QJfYXiRWY=
github.com/wabarc/screenshot v1.3.2-0.20210824153650-d47a1474a43e/go.mod h1:ei8rqXW5mdztkqcsb81YUVwBZFROgyjAQJrEKEiMWfY=
github.com/wabarc/telegra.ph v0.0.0-20210822083402-82f95ce60a37 h1:lqVzAnARDEJO+bTtQiCRu2lg55QxboohtP/RZN9Y9mU=
github.com/wabarc/telegra.ph v0.0.0-20210822083402-82f95ce60a37/go.mod h1:532VM0F+WU2TSVvolJN3U5xihKLBt8ubLqyfWExM7As=
github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c h1:HPTqoafOgXbqEMD54v3cTTJKKK3fd79UqG1ILR9S640=
github.com/wabarc/warcraft v0.1.1-0.20210711171056-a5eec617b86c/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc=
github.com/wabarc/wbipfs v0.2.0 h1:TksxONxL/LrBGZ6TTcWD4p+FAzuQPfLQyCL7K64isW8=
Expand Down Expand Up @@ -862,9 +856,9 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down Expand Up @@ -1076,8 +1070,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
maunium.net/go/maulogger/v2 v2.2.4/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A=
maunium.net/go/mautrix v0.9.19 h1:8ZoDuijJOKxgEOMDoBN2B6at0Ba7EJpsqWA/5jV7ELw=
maunium.net/go/mautrix v0.9.19/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=
maunium.net/go/mautrix v0.9.20 h1:MUKnH3etwsoi1K/1jhUg1NG8qcupzrj3w56rDnDF7mc=
maunium.net/go/mautrix v0.9.20/go.mod h1:7IzKfWvpQtN+W2Lzxc0rLvIxFM3ryKX6Ys3S/ZoWbg8=
mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=
Expand Down
23 changes: 18 additions & 5 deletions pooling/pooling.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,20 @@ import (
"sync"
"time"

"github.com/phf/go-queue/queue"
"github.com/wabarc/logger"
"github.com/wabarc/wayback/errors"
)

const maxTime = 5 * time.Minute
var maxTime = 5 * time.Minute

var (
ErrPoolNotExist = errors.New("pool not exist") // ErrPoolNotExist pool not exist
ErrTimeout = errors.New("process timeout") // ErrTimeout process timeout
)

var q = queue.New()

type resource struct {
id int
}
Expand Down Expand Up @@ -49,8 +52,13 @@ func New(size int) Pool {

// Roll wrapper service as function to the resource pool.
func (p Pool) Roll(service func()) {
do := func(service func(), wg *sync.WaitGroup) {
do := func(wg *sync.WaitGroup) {
defer wg.Done()
fn, ok := q.PopBack().(func())
if !ok {
logger.Error("pop service failed")
return
}

r, err := p.pull()
defer p.push(r)
Expand All @@ -60,13 +68,18 @@ func (p Pool) Roll(service func()) {
}
logger.Debug("roll service on #%d", r.id)

logger.Debug("roll service func: %#v", service)
service()
logger.Debug("roll service func: %#v", fn)
fn()
}

// Inserts a new value service at the front of queue q.
q.PushFront(service)

var wg sync.WaitGroup
wg.Add(1)
go do(service, &wg)

// TODO: retry
go do(&wg)
wg.Wait()
}

Expand Down
49 changes: 49 additions & 0 deletions pooling/pooling_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright 2021 Wayback Archiver. All rights reserved.
// Use of this source code is governed by the GNU GPL v3
// license that can be found in the LICENSE file.

package pooling // import "github.com/wabarc/wayback/pooling"

import (
"runtime"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/wabarc/logger"
)

func TestTimeout(t *testing.T) {
logger.SetLogLevel(logger.LevelFatal)

maxTime = time.Microsecond

c := 2
p := New(c)

var i int32
var wg sync.WaitGroup
for i < 5 {
wg.Add(1)
p.Roll(func() {
time.Sleep(time.Millisecond)
})
wg.Done()
atomic.AddInt32(&i, 1)
runtime.Gosched()
}
wg.Wait()

if len(p) != c {
t.Fatalf("The length of pool got %d instead of %d", len(p), c)
}

p.Roll(func() {
time.Sleep(time.Millisecond)
})

if len(p) != c {
t.Fatalf("The length of pool got %d instead of %d", len(p), c)
}
}
20 changes: 11 additions & 9 deletions reduxer/reduxer.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,15 +230,17 @@ func Capture(ctx context.Context, urls ...string) (shots []screenshot.Screenshot

// Asset returns paths of asset on the local disk.
func (b *Bundle) Asset() (paths []Asset) {
logger.Debug("assets: %#v", b.Assets)
paths = []Asset{
b.Assets.Img,
b.Assets.PDF,
b.Assets.Raw,
b.Assets.Txt,
b.Assets.HAR,
b.Assets.WARC,
b.Assets.Media,
if b != nil {
logger.Debug("assets: %#v", b.Assets)
paths = []Asset{
b.Assets.Img,
b.Assets.PDF,
b.Assets.Raw,
b.Assets.Txt,
b.Assets.HAR,
b.Assets.WARC,
b.Assets.Media,
}
}
return
}
Expand Down
2 changes: 1 addition & 1 deletion template/render/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (gh *GitHub) ForPublish() *Render {

const tmpl = `{{range $ := .}}**[{{ $.Arc | name }}]({{ $.Ext | extra }})**:
> source: [{{ $.Src | unescape | revert }}]({{ $.Src | revert }})
> archived: {{ if $.Dst | isURL }}[{{ $.Dst | unescape }}]({{ $.Dst }})
> archived: {{ if $.Dst | isURL }}[{{ $.Dst | unescape }}]({{ $.Dst | escapeString }})
{{ else }}{{ $.Dst }}
{{ end }}
{{ end }}`
Expand Down
4 changes: 2 additions & 2 deletions template/render/matrix.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func (m *Matrix) ForReply() *Render {
var tmplBytes bytes.Buffer

const tmpl = `{{range $ := .}}<b><a href='{{ $.Ext | extra }}'>{{ $.Arc | name }}</a></b>:<br>
• <a href="{{ $.Src | revert }}">source</a> - {{ $.Dst }}<br>
• <a href="{{ $.Src | revert }}">source</a> - {{ $.Dst | escapeString }}<br>
<br>
{{ end }}`

Expand Down Expand Up @@ -70,7 +70,7 @@ func (m *Matrix) ForPublish() *Render {
}

const tmpl = `{{range $ := .}}<b><a href='{{ $.Ext | extra }}'>{{ $.Arc | name }}</a></b>:<br>
• <a href="{{ $.Src | revert }}">source</a> - {{ $.Dst }}<br>
• <a href="{{ $.Src | revert }}">source</a> - {{ $.Dst | escapeString }}<br>
<br>
{{ end }}`

Expand Down
2 changes: 2 additions & 0 deletions template/render/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/wabarc/wayback"
"github.com/wabarc/wayback/config"
"github.com/wabarc/wayback/reduxer"
"golang.org/x/net/html"
)

const (
Expand Down Expand Up @@ -59,6 +60,7 @@ func (r *Render) String() string {
func funcMap() template.FuncMap {
cache := "https://webcache.googleusercontent.com/search?q=cache:"
return template.FuncMap{
"escapeString": html.EscapeString,
"unescape": func(link string) string {
unescaped, err := url.QueryUnescape(link)
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions template/render/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ type Telegram struct {
func (t *Telegram) ForReply() (r *Render) {
var tmplBytes bytes.Buffer

const tmpl = `{{range $ := .}}<b><a href='{{ $.Ext | extra }}'>{{ $.Arc | name }}</a></b>:
const tmpl = `{{range $ := .}}<b><a href="{{ $.Ext | extra }}">{{ $.Arc | name }}</a></b>:
{{ range $map := $.Dst -}}
{{ range $src, $dst := $map -}}
• <a href="{{ $src | revert }}">source</a> - {{ if $dst | isURL }}<a href="{{ $dst }}">{{ $dst }}</a>{{ else }}{{ $dst }}{{ end }}
• <a href="{{ $src | revert }}">source</a> - {{ if $dst | isURL }}<a href="{{ $dst }}">{{ $dst }}</a>{{ else }}{{ $dst | escapeString }}{{ end }}
{{ end }}{{ end }}
{{ end }}`

Expand Down Expand Up @@ -73,8 +73,8 @@ func (t *Telegram) ForPublish() (r *Render) {
}

tmpl := `{{range $ := .}}
<b><a href='{{ $.Ext | extra }}'>{{ $.Arc | name }}</a></b>:
• <a href="{{ $.Src | revert }}">source</a> - {{ if $.Dst | isURL }}<a href="{{ $.Dst }}">{{ $.Dst }}</a>{{ else }}{{ $.Dst }}{{ end }}
<b><a href="{{ $.Ext | extra }}">{{ $.Arc | name }}</a></b>:
• <a href="{{ $.Src | revert }}">source</a> - {{ if $.Dst | isURL }}<a href="{{ $.Dst }}">{{ $.Dst }}</a>{{ else }}{{ $.Dst | escapeString }}{{ end }}
{{ end }}`

tpl, err := template.New("message").Funcs(funcMap()).Parse(tmpl)
Expand Down
Loading

0 comments on commit d5bead8

Please sign in to comment.