init

.gitignore

@@ -0,0 +1,5 @@
+.DS_Store
+.vscode
+.env
+parser/domains.csv
+dist

.goreleaser.yml

@@ -0,0 +1,68 @@
+# set environment variables to be used in the build process
+env:
+    # enable GO111Modules if you are using Go1.11 modules in your project for dependency management
+    - GO111MODULE=on
+    # Disable CGO - We will go more into this later
+    - CGO_ENABLED=0
+# before are hooks that will be run before any builds are done, so good to put install scripts and stuff that your builds need here
+before:
+    hooks:
+        # Remove unused packaged from the build process
+        - go mod tidy
+        # You may remove this if you don't use go modules - Downloads all modules specified in go.mod
+        - go mod download
+# buids. This is very basic, but there is a ton of customization you can do here.
+# I would check out https://goreleaser.com/customization/ to learn more.
+builds:
+    - env:
+          - CGO_ENABLED=0
+      # your entrypoint into your application
+      main: ./main.go
+      # the name of the binary that you want to generate - this will also be the binary name for when we get to homebrew...
+      # Example: goclone <url> where goclone is the name of the binary
+      binary: goclone
+# The different tar archives to build for - includes naming customization and such :)
+archives:
+    # default format for Mac (darwin) and Linux
+    - format: tar.gz
+      # if the OS is windows, we override and put everything in a .zip instead.
+      format_overrides:
+          - goos: windows
+            format: zip
+      # Just the naming convention for tarballs... Example: msconsole_1.0.0_darwin_64bit
+      name_template: '{{.ProjectName}}_{{.Version}}_{{.Os}}-{{.Arch}}'
+      # Replace the cryptic arch's with readable human naming conventions
+      replacements:
+          amd64: 64bit
+          386: 32bit
+          arm: ARM
+          arm64: ARM64
+          darwin: macOS
+          linux: Linux
+          windows: Windows
+      files:
+          - README.md
+# The generated checksum after your build. No need to touch this unless you know what you are doing.
+checksum:
+    name_template: 'checksums.txt'
+# The name of the reason snapshot
+snapshot:
+    name_template: '{{ .Tag }}-next'
+# GoReleaser will generate you a changelog depending on the commits that you have created.
+changelog:
+    sort: asc
+    filters:
+        exclude:
+            - '^docs:'
+            - '^test:'
+            - 'README'
+            - Merge pull request
+            - Merge branch
+# Seperate repo that you need to create before you deploy.
+brews:
+    - github:
+          owner: imthaghost
+          name: homebrew-goclone
+      folder: Formula
+      homepage: https://github.com/imthaghost/goclone
+      description: Website Cloner - Utilizes powerful go routines to clone websites to your computer within seconds.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 None
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,75 @@
+<p align="center">
+  <a href="https://goclone.herokuapp.com/">
+    <img alt="jedi" src="docs/media/logo.png"> 
+  </a>
+</p>
+<p align="center">
+Copy websites to your computer! goclone is a utility that allows you to download a website from the Internet to a local directory. Get html, css, js, images, and other files from the server to your computer. goclone arranges the original site's relative link-structure. Simply open a page of the "mirrored" website in your browser, and you can browse the site from link to link, as if you were viewing it online.
+</p>
+<br>
+<p align="center"><a href="https://goclone.herokuapp.com/">Official Website</a></p>
+<br>
+<p align="center">
+   <a href="https://goreportcard.com/report/github.com/imthaghost/goclone"><img src="https://goreportcard.com/badge/github.com/imthaghost/goclone"></a>
+   <a href="https://github.com/imthaghost/gitmoji-changelog">
+    <img src="https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg"alt="gitmoji-changelog">
+  </a>
+</p>
+<br>
+
+![Example](/docs/media/cloner.gif)
+
+## Table of Contents
+
+-   [Installation](#installation)
+-   [Examples](#examples)
+-   [License](#license)
+-   [Contributors](#contributors)
+
+## 🚀 Installation
+
+### Brew
+
+```bash
+# tap
+brew tap imthaghost/goclone
+# install tool
+brew install goclone
+```
+
+### Manual
+
+```bash
+# go get :)
+go get https://github.com/imthaghost/goclone
+# change to project directory using your GOPATH
+cd $GOPATH/src/github.com/imthaghost/goclone
+# build and install application
+go install
+```
+
+## Examples
+
+###
+
+```bash
+# goclone <url>
+goclone https://dribbble.com
+```
+
+![Dribbble](/docs/media/dribbble.gif)
+
+## 📝 License
+
+By contributing, you agree that your contributions will be licensed under its MIT License.
+
+In short, when you submit code changes, your submissions are understood to be under the same [MIT License](http://choosealicense.com/licenses/mit/) that covers the project. Feel free to contact the maintainers if that's a concern.
+
+## Contributors
+
+Contributions are welcome! Please see [Contributing Guide](https://imthaghost/zeus) for more details.
+
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/imthaghost"><img src="https://avatars3.githubusercontent.com/u/46610773?s=460&v=4" width="75px;" alt="Gary Frederick"/><br /><sub><b>Tha Ghost</b></sub></a><br /><a href="https://github.com/imthaghost/goclone/commits?author=imthaghost" title="Code">💻</a></td>
+  </tr>

crawler/collector.go

@@ -0,0 +1,60 @@
+package crawler
+
+import (
+	"fmt"
+
+	"github.com/gocolly/colly"
+)
+
+// Collector searches for css, js, and images within a given link
+func Collector(url string, projectPath string) {
+	// create a new collector
+	c := colly.NewCollector(
+		// asychronus boolean
+		colly.Async(true),
+	)
+	// search for all link tags that have a rel attribute that is equal to stylesheet - CSS
+	c.OnHTML("link[rel='stylesheet']", func(e *colly.HTMLElement) {
+		// hyperlink reference
+		link := e.Attr("href")
+		// print css file was found
+		fmt.Println("Css found", "-->", link)
+		// extraction
+		Extractor(e.Request.AbsoluteURL(link), projectPath)
+	})
+	// search for all script tags with src attribute -- JS
+	c.OnHTML("script[src]", func(e *colly.HTMLElement) {
+		// src attribute
+		link := e.Attr("src")
+		// Print link
+		fmt.Println("Js found", "-->", link)
+		// extraction
+		Extractor(e.Request.AbsoluteURL(link), projectPath)
+	})
+	// serach for all img tags with src attribute -- Images
+	c.OnHTML("img[src]", func(e *colly.HTMLElement) {
+		// src attribute
+		link := e.Attr("src")
+		// Print link
+		fmt.Println("Img found", "-->", link)
+		// extraction
+		Extractor(e.Request.AbsoluteURL(link), projectPath)
+	})
+	//Before making a request
+	c.OnRequest(func(r *colly.Request) {
+		link := r.URL.String()
+		if url == link {
+			HTMLExtractor(link, projectPath)
+		}
+	})
+	// Response of each visited page
+	// c.OnResponse(func(r *colly.Response) {
+	// 	link := r.Ctx.Get("url")
+	// 	// check if the url being visited is the root for searching if so write it as a page
+	// 	if url == link {
+	// 		HTMLExtractor(link, projectPath)
+	// 	}
+	// })
+	c.Visit(url)
+	c.Wait()
+}

crawler/crawler.go

@@ -0,0 +1,7 @@
+package crawler
+
+// Crawl asks the neccessary crawlers for collecting links for building the web page
+func Crawl(site string, projectPath string) {
+	// searches for css, js, and images within a given link
+	Collector(site, projectPath)
+}