diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..69e8150 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +ko_fi: dwisiswant0 +custom: ["https://www.buymeacoffee.com/dw1", "https://paypal.me/dw1s", "https://saweria.co/dwisiswant0"] diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..d976cf2 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,27 @@ +name: Release +on: + create: + tags: + - v* + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v2 + with: + go-version: 1.11 + + - name: "Create release on GitHub" + uses: goreleaser/goreleaser-action@v2 + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + args: "release --rm-dist" + version: latest diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d26954d --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,21 @@ +builds: + - binary: unew + main: main.go + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + +archives: + - id: tgz + format: tar.gz + replacements: + darwin: macOS + format_overrides: + - goos: windows + format: zip diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6114a31 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 dwisiswant0 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb5d840 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# unew + +**u**_**(rl)**_**new** — A tool for append URLs, skipping duplicates & combine parameters. Inspired by [anew](https://github.com/tomnomnom/anew) & [qsreplace](https://github.com/tomnomnom/qsreplace). + +# Usage + +```bash +▶ cat urls.txt | unew +# or +▶ unew urls.txt +# or, save the results +▶ unew urls.txt output.txt +``` + +## Flags + +Usage of `unew`: +``` + -combine + Combine parameters + -r string + Replace parameters value +``` + +# Install + +with [Go](https://golang.org/doc/install): + +```bash +▶ go get -u github.com/dwisiswant0/unew +``` + +# Workaround + +If you have a list as + +```txt +https://twitter.com/dwisiswant0?href=evilzone.org +https://twitter.com/dwisiswant0 +https://twitter.com/dwisiswant0?ref=github&utm_source=github +https://www.linkedin.com/in/dwisiswanto/ +https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id +https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github +``` + +Sample workaround +```bash +▶ cat urls.txt | unew +https://twitter.com/dwisiswant0?href=evilzone.org +https://www.linkedin.com/in/dwisiswanto/ +``` + +If the list contains multiple URLs with same path, it will save the first one and its parameters. + +But you can combine parameters if the same path exists by using `-combine` flag. + +```bash +▶ cat urls.txt | unew -combine +https://twitter.com/dwisiswant0?href=evilzone.org&ref=github&utm_source=github +https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=id&utm_medium=github +``` + +Use the `-r` flag if you want to change the value of all parameters. + +```bash +▶ cat urls.txt | unew -combine -r "/etc/passwd" +https://twitter.com/dwisiswant0?href=%2Fetc%2Fpasswd&ref=%2Fetc%2Fpasswd&utm_source=%2Fetc%2Fpasswd +https://www.linkedin.com/in/dwisiswanto/?originalSubdomain=%2Fetc%2Fpasswd&utm_medium=%2Fetc%2Fpasswd +``` \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..883f6f0 --- /dev/null +++ b/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "net/url" + "os" + "strings" +) + +var combine bool +var outfile *os.File +var sc *bufio.Scanner +var replace, outtext, fr, fs string + +func init() { + flag.StringVar(&replace, "r", "", "Replace parameters value") + flag.BoolVar(&combine, "combine", false, "Combine parameters") + flag.Parse() + + fr := flag.Arg(0) + + if isStdin() { + sc = bufio.NewScanner(os.Stdin) + } else if fr != "" { + r, err := os.Open(fr) + if err == nil { + sc = bufio.NewScanner(r) + } + } else { + os.Exit(1) + } +} + +func main() { + urls := make(map[string]string) + + fs := flag.Arg(1) + if fs != "" { + outfile, _ = os.OpenFile(fs, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + defer outfile.Close() + } + + for sc.Scan() { + u, err := url.ParseRequestURI(sc.Text()) + if err != nil { + continue + } + + b := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path) + + if _, d := urls[b]; d { + if combine { + q := []string{urls[b], u.RawQuery} + u.RawQuery = remDup(strings.Join(q, "&")) + } else { + continue + } + } else { + urls[b] = u.RawQuery + } + + if replace != "" { + u.RawQuery = qsReplace(u.Query(), replace) + } + + if !combine { + outtext = fmt.Sprintf("%s%s\n", b, qMark(u.RawQuery)) + fmt.Printf("%s", outtext) + if fs != "" { + fmt.Fprintf(outfile, "%s", outtext) + } + + continue + } + + urls[b] = u.RawQuery + } + + if !combine { + return + } + + for k, v := range urls { + outtext = fmt.Sprintf("%s%s\n", k, qMark(v)) + fmt.Printf("%s", outtext) + if fs != "" { + fmt.Fprintf(outfile, "%s", outtext) + } + } +} + +func isStdin() bool { + f, e := os.Stdin.Stat() + if e != nil { + return false + } + + if f.Mode()&os.ModeNamedPipe == 0 { + return false + } + + return true +} + +func qMark(q string) string { + if q == "" { + return "" + } + + return "?" + q +} + +func remDup(q string) string { + qs := url.Values{} + ps, _ := url.ParseQuery(q) + + for p, v := range ps { + for range v { + v = v[:1:1] + qs.Set(p, v[0]) + } + } + + return qs.Encode() +} + +func qsReplace(q url.Values, r string) string { + qs := url.Values{} + for p := range q { + qs.Set(p, r) + } + + return qs.Encode() +}