Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integration with Ghostarchive #497

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Integration with Ghostarchive #497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0d9cd5d
Add support Ghostarchive
waybackarchiver Feb 29, 2024
4f55ea6
Update minimal go version for testing workflow
waybackarchiver Feb 29, 2024
afb69fd
Bump minimal go version for Dockerfile
waybackarchiver Mar 1, 2024
de5c104
Update dependencies
waybackarchiver Jun 16, 2024
1c377d9
Remove toolchain directive
waybackarchiver Jun 22, 2024
5076173
Fix test
waybackarchiver Jun 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/testing.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
# The step `Set up Chrome` do not support arm64 yet, so specified s version to use amd64 arch
# Refs to https://github.com/actions/runner-images#available-images
os: [ ubuntu-24.04, macos-13, windows-2022 ]
go: [ "1.19", "1.20", "1.21", "1.22" ]
go: [ "1.21", "1.22" ]
include:
# only update test coverage stats with the most recent go version on linux
- go: 1.x
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# license that can be found in the LICENSE file.
#
# syntax=docker/dockerfile:1.2
ARG GO_VERSION=1.20
ARG GO_VERSION=1.22

FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder
COPY --from=tonistiigi/xx:golang / /
Expand Down
2 changes: 1 addition & 1 deletion build/docker/Dockerfile.all
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1.2
ARG GO_VERSION=1.20
ARG GO_VERSION=1.22

FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder
COPY --from=tonistiigi/xx:golang / /
Expand Down
2 changes: 1 addition & 1 deletion build/docker/Dockerfile.dev
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1.2
ARG GO_VERSION=1.20
ARG GO_VERSION=1.22

FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder

Expand Down
2 changes: 1 addition & 1 deletion build/redhat/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Use of this source code is governed by the GNU GPL v3
# license that can be found in the LICENSE file.
#
FROM golang:1.20-alpine AS builder
FROM golang:1.22-alpine AS builder

Check warning

Code scanning / Scorecard

Pinned-Dependencies Medium

score is 7: containerImage not pinned by hash
Click Remediation section below to solve this issue

ARG WAYBACK_IPFS_APIKEY

Expand Down
8 changes: 7 additions & 1 deletion cmd/wayback/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ var (
is bool
ip bool
ph bool
ga bool

daemon []string

Expand Down Expand Up @@ -73,6 +74,7 @@ func init() {
rootCmd.Flags().BoolVarP(&is, "is", "", true, "Wayback webpages to Archive Today")
rootCmd.Flags().BoolVarP(&ip, "ip", "", true, "Wayback webpages to IPFS")
rootCmd.Flags().BoolVarP(&ph, "ph", "", true, "Wayback webpages to Telegraph")
rootCmd.Flags().BoolVarP(&ga, "ga", "", true, "Wayback webpages to Ghostarchive")
rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc")
rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs")
rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port")
Expand Down Expand Up @@ -128,6 +130,9 @@ func setToEnv(cmd *cobra.Command) {
if flags.Changed("ph") {
os.Setenv("WAYBACK_ENABLE_PH", fmt.Sprint(ph))
}
if flags.Changed("ga") {
os.Setenv("WAYBACK_ENABLE_GA", fmt.Sprint(ga))
}
if flags.Changed("token") {
os.Setenv("WAYBACK_TELEGRAM_TOKEN", token)
}
Expand All @@ -153,11 +158,12 @@ func setToEnv(cmd *cobra.Command) {

// nolint:gocyclo
func run(cmd *cobra.Command, args []string) {
if !ia && !is && !ip && !ph {
if !ia && !is && !ip && !ph && !ga {
ia, is, ip = true, true, true
os.Setenv("WAYBACK_ENABLE_IA", "true")
os.Setenv("WAYBACK_ENABLE_IS", "true")
os.Setenv("WAYBACK_ENABLE_IP", "true")
os.Setenv("WAYBACK_ENABLE_GA", "true")
}

setToEnv(cmd)
Expand Down
3 changes: 3 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const (
SLOT_IS = "is" // archive.today
SLOT_IP = "ip" // IPFS
SLOT_PH = "ph" // Telegraph
SLOT_GA = "ga" // Ghostarchive
SLOT_TT = "tt" // Time Travel
SLOT_GC = "gc" // Google Cache

Expand Down Expand Up @@ -68,6 +69,7 @@ func SlotName(s string) string {
SLOT_IS: "archive.today",
SLOT_IP: "IPFS",
SLOT_PH: "Telegraph",
SLOT_GA: "Ghostarchive",
SLOT_TT: "Time Travel",
SLOT_GC: "Google Cache",
}
Expand All @@ -86,6 +88,7 @@ func SlotExtra(s string) string {
SLOT_IS: "https://archive.today/",
SLOT_IP: "https://ipfs.github.io/public-gateway-checker/",
SLOT_PH: "https://telegra.ph/",
SLOT_GA: "https://ghostarchive.org/",
SLOT_TT: "http://timetravel.mementoweb.org/",
SLOT_GC: "https://webcache.googleusercontent.com/",
}
Expand Down
5 changes: 4 additions & 1 deletion config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,7 @@ func TestEnableSlots(t *testing.T) {
os.Setenv("WAYBACK_ENABLE_IS", "true")
os.Setenv("WAYBACK_ENABLE_IP", "true")
os.Setenv("WAYBACK_ENABLE_PH", "true")
os.Setenv("WAYBACK_ENABLE_GA", "true")

parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
Expand All @@ -409,10 +410,11 @@ func TestEnableSlots(t *testing.T) {
SLOT_IS: true,
SLOT_IP: true,
SLOT_PH: true,
SLOT_GA: true,
}
got := opts.Slots()

if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] {
if got == nil || !got[SLOT_IA] || !got[SLOT_IS] || !got[SLOT_IP] || !got[SLOT_PH] || !got[SLOT_GA] {
t.Fatalf(`Unexpected default slots, got %v instead of %v`, got, expected)
}
}
Expand All @@ -431,6 +433,7 @@ func TestDefaultSlots(t *testing.T) {
SLOT_IS: true,
SLOT_IP: true,
SLOT_PH: true,
SLOT_GA: true,
}
got := opts.Slots()

Expand Down
2 changes: 2 additions & 0 deletions config/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const (
defEnabledIS = true
defEnabledIP = true
defEnabledPH = true
defEnabledGA = true

defTelegramToken = ""
defTelegramChannel = ""
Expand Down Expand Up @@ -286,6 +287,7 @@ func NewOptions() *Options {
SLOT_IS: defEnabledIS,
SLOT_IP: defEnabledIP,
SLOT_PH: defEnabledPH,
SLOT_GA: defEnabledGA,
},
telegram: &telegram{
token: defTelegramToken,
Expand Down
2 changes: 1 addition & 1 deletion docker-bake.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Go version
variable "GO_VERSION" {
default = "1.20"
default = "1.22"
}

variable "WAYBACK_IPFS_APIKEY" {
Expand Down
46 changes: 25 additions & 21 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
module github.com/wabarc/wayback

// +heroku goVersion go1.20
// +heroku goVersion go1.21

go 1.20
go 1.21

require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/PuerkitoBio/goquery v1.9.0
github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
github.com/cretz/bine v0.2.0
github.com/davecgh/go-spew v1.1.1
Expand All @@ -28,28 +28,29 @@ require (
github.com/nbd-wtf/go-nostr v0.17.1-0.20230426111250-32ca737acf77
github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.18.0
github.com/prometheus/common v0.46.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/common v0.48.0
github.com/rs/xid v1.4.0
github.com/slack-go/slack v0.11.2
github.com/spf13/cobra v1.6.1
github.com/wabarc/archive.is v1.4.0
github.com/wabarc/archive.org v1.2.1-0.20210708220121-cb9b83ff9896
github.com/wabarc/ghostarchive v0.1.1
github.com/wabarc/go-anonfile v0.1.0
github.com/wabarc/go-catbox v0.1.0
github.com/wabarc/helper v0.0.0-20230418130954-be7440352bcb
github.com/wabarc/imgbb v1.0.0
github.com/wabarc/ipfs-pinner v1.1.1-0.20230502052510-dc378f9e202b
github.com/wabarc/logger v0.0.0-20210730133522-86bd3f31e792
github.com/wabarc/playback v0.0.0-20230331122619-84484ab4d599
github.com/wabarc/playback v0.0.0-20240229124108-a6ddc935de3c
github.com/wabarc/proxier v0.0.0-20230610135141-b55fe1536465
github.com/wabarc/rivet v0.1.4-0.20230505152228-2c5c81b4bd10
github.com/wabarc/screenshot v1.6.1-0.20240214000834-3820163034f4
github.com/wabarc/telegra.ph v0.0.0-20230318134541-a0922e1ace3a
github.com/wabarc/warcraft v0.3.1-0.20230308125707-3daa5592ba52
go.etcd.io/bbolt v1.3.6
golang.org/x/net v0.21.0
golang.org/x/sync v0.6.0
golang.org/x/net v0.26.0
golang.org/x/sync v0.7.0
gopkg.in/irc.v4 v4.0.0
gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3
maunium.net/go/mautrix v0.12.0
Expand All @@ -62,7 +63,7 @@ require (
github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect
github.com/SaveTheRbtz/generic-sync-map-go v0.0.0-20230201052002-6c5833b989be // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand All @@ -77,6 +78,7 @@ require (
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect
github.com/chromedp/chromedp v0.9.5 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
Expand All @@ -85,15 +87,14 @@ require (
github.com/dop251/goja v0.0.0-20221115122301-6c0d9883792e // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/fortytw2/leaktest v1.3.0 // indirect
github.com/gaukas/godicttls v0.0.3 // indirect
github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.3.2 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/iawia002/lia v0.0.0-20221116085912-1f653221be4b // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
Expand All @@ -107,7 +108,7 @@ require (
github.com/kallydev/telegraph-go v1.0.1-0.20230318133700-df034d9eed50 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/kkdai/youtube/v2 v2.7.18 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
Expand All @@ -132,9 +133,12 @@ require (
github.com/multiformats/go-multistream v0.5.0 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/oliamb/cutter v0.2.2 // indirect
github.com/onsi/ginkgo/v2 v2.19.0 // indirect
github.com/onsi/gomega v1.33.1 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/refraction-networking/utls v1.3.2 // indirect
github.com/quic-go/quic-go v0.45.0 // indirect
github.com/refraction-networking/utls v1.6.3 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
Expand All @@ -151,14 +155,14 @@ require (
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/ybbus/httpretry v1.0.2 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
golang.org/x/tools v0.17.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.22.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
Expand Down