v1.0.7 Release preparation

.github/workflows/build-test.yml

@@ -13,12 +13,16 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.17
 
       - name: Test
         run: go test .
         working-directory: cmd/dnsx/
 
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
+
       - name: Build
         run: go build .
         working-directory: cmd/dnsx/

.github/workflows/dockerhub-push.yml

@@ -1,34 +1,40 @@
 name: 🌥 Docker Push
 
 on:
-  release:
-    types: [published]
+  workflow_run:
+    workflows: ["🎉 Release Binary"]
+    types:
+      - completed
   workflow_dispatch:
 
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Git Checkout
         uses: actions/checkout@v2
-      -
-        name: Set up QEMU
+
+      - name: Get Github tag
+        id: meta
+        run: |
+          echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/dnsx/releases/latest" | jq -r .tag_name)"
+
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
+
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      -
-        name: Login to DockerHub
+
+      - name: Login to DockerHub
         uses: docker/login-action@v1 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-      -
-        name: Build and push
+
+      - name: Build and push
         uses: docker/build-push-action@v2
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
-          tags: projectdiscovery/dnsx:latest
+          tags: projectdiscovery/dnsx:latest,projectdiscovery/dnsx:${{ steps.meta.outputs.tag }}

.github/workflows/functional-test.yml

@@ -0,0 +1,25 @@
+name: 🧪 Functional Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  functional:
+    name: Functional Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Functional Tests
+        run: |
+          chmod +x run.sh
+          bash run.sh
+        working-directory: cmd/functional-test

.github/workflows/release-binary.yml

@@ -18,7 +18,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.16
+          go-version: 1.17
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -1,3 +1,8 @@
 cmd/dnsx/dnsx
 .DS_Store
-dist/
+dist/
+cmd/functional-test/dnsx_dev
+cmd/functional-test/functional-test
+cmd/functional-test/dnsx
+cmd/functional-test/log.txt
+cmd/functional-test/*.cfg

.goreleaser.yml

@@ -1,21 +1,35 @@
+before:
+  hooks:
+    - go mod tidy
+
 builds:
-    - binary: dnsx
-      main: cmd/dnsx/dnsx.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+    - goos: windows
+      goarch: 'arm64'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/dnsx/dnsx.go
+
 archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

Dockerfile

@@ -1,5 +1,5 @@
-FROM golang:1.16.7-alpine3.14 AS build-env
-RUN go get -v github.com/projectdiscovery/dnsx/cmd/dnsx
+FROM golang:1.17.3-alpine3.14 AS build-env
+RUN go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest
 
 FROM alpine:3.14
 RUN apk add --no-cache bind-tools ca-certificates

README.md

@@ -48,6 +48,15 @@
  - Handles wildcard subdomains in automated way.
  - **Stdin** and **stdout** support to work with other tools.
 
+# Installation Instructions
+
+
+dnsx requires **go1.17** to install successfully. Run the following command to get the repo -
+
+```sh
+go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest
+```
+
 # Usage
 
 ```sh
@@ -56,44 +65,52 @@ dnsx -h
 
 This will display help for the tool. Here are all the switches it supports.
 
-| Flag                | Description                        | Example               |
-| ------------------- | ---------------------------------- | --------------------- |
-| a                   | Query A record                     | dnsx -a               |
-| aaaa                | Query AAAA record                  | dnsx -aaaa            |
-| cname               | Query CNAME record                 | dnsx -cname           |
-| ns                  | Query NS record                    | dnsx -ns              |
-| ptr                 | Query PTR record                   | dnsx -ptr             |
-| txt                 | Query TXT record                   | dnsx -txt             |
-| mx                  | Query MX record                    | dnsx -mx              |
-| soa                 | Query SOA record                   | dnsx -soa             |
-| raw                 | Operates like dig                  | dnsx -raw             |
-| rcode               | DNS Response codes                 | dnsx -rcode 0,1,2     |
-| l                   | File input list of subdomains/host | dnsx -l list.txt      |
-| json                | JSON output                        | dnsx -json            |
-| r                   | File or comma separated resolvers  | dnsx -r 1.1.1.1       |
-| rl                  | Limit of DNS request/second        | dnsx -rl 100          |
-| resp                | Display response data              | dnsx -cname -resp     |
-| resp-only           | Display only response data         | dnsx -cname resp-only |
-| retry               | Number of DNS retries              | dnsx -retry 1         |
-| silent              | Show only results in the output    | dnsx -silent          |
-| stats               | Display stats of the running scan  | dnsx -stats           |
-| o                   | File to write output to (optional) | dnsx -o output.txt    |
-| t                   | Concurrent threads to make         | dnsx -t 100           |
-| trace               | Perform dns trace                  | dnsx -trace           |
-| trace-max-recursion | Max recursion for dns trace        | dnsx -t 32767         |
-| verbose             | Verbose output                     | dnsx -verbose         |
-| version             | Show version of dnsx               | dnsx -version         |
-| wd                  | Wildcard domain name for filtering | dnsx -wd example.com  |
-| wt                  | Wildcard Filter Threshold          | dnsx -wt 5            |
-
-
-# Installation Instructions
-
-
-dnsx requires **go1.16+** to install successfully. Run the following command to get the repo -
-
-```sh
-go get -v github.com/projectdiscovery/dnsx/cmd/dnsx
+```console
+INPUT:
+   -l, -list string  File input with list of sub(domains)/hosts
+
+QUERY:
+   -a      Query A record (default)
+   -aaaa   Query AAAA record
+   -cname  Query CNAME record
+   -ns     Query NS record
+   -txt    Query TXT record
+   -ptr    Query PTR record
+   -mx     Query MX record
+   -soa    Query SOA record
+
+FILTERS:
+   -resp               Display DNS response
+   -resp-only          Display DNS response only
+   -rcode, -rc string  Display DNS status code (eg. -rcode noerror,servfail,refused)
+
+RATE-LIMIT:
+   -t, -c int            Number of concurrent threads to use (default 100)
+   -rl, -rate-limit int  Number of DNS request/second (disabled as default) (default -1)
+
+OUTPUT:
+   -o, -output string  File to write output (optional)
+   -json               Write output in JSONL(ines) format
+
+DEBUG:
+   -silent       Show only results in the output
+   -v, -verbose  Verbose output
+   -raw, -debug  Display RAW DNS response
+   -stats        Display stats of the running scan
+   -version      Show version of dnsx
+
+OPTIMIZATION:
+   -retry int                Number of DNS retries (default 1)
+   -hf, -hostsfile           Parse system host file
+   -trace                    Perform DNS trace
+   -trace-max-recursion int  Max recursion for dns trace (default 32767)
+   -flush-interval int       Flush interval of output file (default 10)
+   -resume                   Resume
+
+CONFIGURATIONS:
+   -r, -resolver string          List of resolvers (file or comma separated)
+   -wt, -wildcard-threshold int  Wildcard Filter Threshold (default 5)
+   -wd, -wildcard-domain string  Domain name for wildcard filtering (other flags will be ignored)
 ```
 
 ### Running dnsx
@@ -136,7 +153,7 @@ support.hackerone.com [104.16.53.111]
 **dnsx** can be used to extract **A** records for the given list of subdomains, for example:-
 
 ```sh
-▶ subfinder -silent -d hackerone.com | dnsx -silent -a -resp-only
+subfinder -silent -d hackerone.com | dnsx -silent -a -resp-only
 
 104.16.99.52
 104.16.100.52

cmd/functional-test/main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/dnsx/internal/testutils"
+)
+
+var (
+	debug   = os.Getenv("DEBUG") == "true"
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainDnsxBinary = flag.String("main", "", "Main Branch Dnsx Binary")
+	devDnsxBinary  = flag.String("dev", "", "Dev Branch Dnsx Binary")
+	testcases      = flag.String("testcases", "", "Test cases file for dnsx functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	var target string
+	if len(parts) > 1 {
+		finalArgs = parts[2:]
+		target = parts[0]
+	}
+	mainOutput, err := testutils.RunDnsxBinaryAndGetResults(target, *mainDnsxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run dnsx main test")
+	}
+	devOutput, err := testutils.RunDnsxBinaryAndGetResults(target, *devDnsxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run dnsx dev test")
+	}
+	if len(mainOutput) == len(devOutput) {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}

cmd/functional-test/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Building functional-test binary'
+go build
+
+echo 'Building DNSX binary from current branch'
+go build -o dnsx_dev ../dnsx
+
+echo 'Installing latest release of DNSX'
+GO111MODULE=on go build -v github.com/projectdiscovery/dnsx/cmd/dnsx
+
+echo 'Starting DNSX functional test'
+./functional-test -main ./dnsx -dev ./dnsx_dev -testcases testcases.txt