Use a random hash instead of generating an url-based filename to store #848
Conversation
LuitelSamikshya
added
the
Type: Bug
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
There was a problem hiding this comment.
The each domain/ip the expected location is in the following format:
outputfolder/domain/hash.txt
The file content of hash.txt should be:
[full URL]
[request]
response
Then a new line to outputfolder/index file should be appended for each retrieved item as follows:
output-folder/domain/hash http://full.url/with/path (status-code status-reason)
This will make the output compatible with: https://github.com/tomnomnom/meg
Note: windows has 255 max char limit for filenames
| data := append([]byte(fmt.Sprintf("[%s]", fullURL)), append([]byte("\n\n"), reqRaw...)...) | ||
| data = append(data, append([]byte("\n"), respRaw...)...) | ||
| fileutil.CreateFolder(domainBaseDir) | ||
| writeErr := os.WriteFile(responsePath, data, 0644) |
Check warning
Code scanning / SonarCloud
<!--SONAR_ISSUE_KEY:AYRS5VF2SgJRRWal-d1K-->Expect WriteFile permissions to be 0600 or less <p>See more on <a href="https://sonarcloud.io/project/issues?id=projectdiscovery_httpx&issues=AYRS5VF2SgJRRWal-d1K&open=AYRS5VF2SgJRRWal-d1K&pullRequest=848">SonarCloud</a></p>
| @@ -91,7 +91,10 @@ | |||
| if err != nil { | |||
| return nil, errors.Wrap(err, "could not create wappalyzer client") | |||
| } | |||
|
|
|||
| if options.StoreResponseDir != "" { | |||
| os.RemoveAll(filepath.Join(options.StoreResponseDir, "index.txt")) | |||
Check warning
Code scanning / SonarCloud
<!--SONAR_ISSUE_KEY:AYRiQyYEW8k5B5vEBcJ1-->Errors unhandled. <p>See more on <a href="https://sonarcloud.io/project/issues?id=projectdiscovery_httpx&issues=AYRiQyYEW8k5B5vEBcJ1&open=AYRiQyYEW8k5B5vEBcJ1&pullRequest=848">SonarCloud</a></p>
| if err != nil { | ||
| gologger.Fatal().Msgf("Could not open/create index file '%s': %s\n", r.options.Output, err) | ||
| } | ||
| defer indexFile.Close() //nolint |
Check warning
Code scanning / SonarCloud
<!--SONAR_ISSUE_KEY:AYRnEvCuNbj9f4MOfSoY-->Deferring unsafe method "Close" on type "*os.File" <p>See more on <a href="https://sonarcloud.io/project/issues?id=projectdiscovery_httpx&issues=AYRnEvCuNbj9f4MOfSoY&open=AYRnEvCuNbj9f4MOfSoY&pullRequest=848">SonarCloud</a></p>
| var err error | ||
| indexPath := filepath.Join(r.options.StoreResponseDir, "index.txt") | ||
| if r.options.Resume { | ||
| indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) |
Check warning
Code scanning / SonarCloud
<!--SONAR_ISSUE_KEY:AYRnEvCuNbj9f4MOfSoX-->Potential file inclusion via variable <p>See more on <a href="https://sonarcloud.io/project/issues?id=projectdiscovery_httpx&issues=AYRnEvCuNbj9f4MOfSoX&open=AYRnEvCuNbj9f4MOfSoX&pullRequest=848">SonarCloud</a></p>
| if r.options.Resume { | ||
| indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) | ||
| } else { | ||
| indexFile, err = os.Create(indexPath) |
Check warning
Code scanning / SonarCloud
<!--SONAR_ISSUE_KEY:AYRnEvCuNbj9f4MOfSoW-->Potential file inclusion via variable <p>See more on <a href="https://sonarcloud.io/project/issues?id=projectdiscovery_httpx&issues=AYRnEvCuNbj9f4MOfSoW&open=AYRnEvCuNbj9f4MOfSoW&pullRequest=848">SonarCloud</a></p>
There was a problem hiding this comment.
$ cat list.txt | go run . -srd test
...
$ cat test/index.txt
test/hackerone.com/b12ceb125f2e005739a0b6d87c900aefc0f97e1ba22609e10ed81ffd9c7b50a8.txt https://hackerone.com:443 (302 Found)
test/projectdiscovery.io/1fca6224ecf596635299a271b83d8a6f060c1f43ba3473895045a5a0cdeea177.txt https://projectdiscovery.io:443 (200 OK)
test/192.168.1.1/c5eb5a4cc76a5cdb16e79864b9ccd26c3553f0c396d0a21bafb7be71c1efcd8c.txt https://192.168.1.1:443 (200 OK)
|
SonarCloud Quality Gate failed. |
Follow up ticket - #873 |
closes #818
Test:
{ "timestamp": "2022-11-02T09:36:57.985867-05:00", "csp": { "domains": [ "https:", "http:", "https://csp.withgoogle.com/csp/gws/cdt1" ] }, "hash": { "body_md5": "703076697ad7b771c974834d230e60f5", "body_mmh3": "-644300001", "body_sha256": "26da22b9175d329a05d0fd071faecfa76f28221c45cb4ea752af8c873db8fac6", "body_simhash": "9809892618964940782", "header_md5": "bff8ac964e4cf59a80e15c28ad2cb5be", "header_mmh3": "-437207301", "header_sha256": "769a560324dae8fe1b056db5b118df68b7146a7904043c6d58209b85f87dfe63", "header_simhash": "9814140865586378174" }, "port": "443", "url": "https://www.google.com:443/search?q=long+url+example&hl=en&rls=en&ei=vmBMY5qoK5LFlwSAmJbABQ&ved=0ahUKEwia_9f5wuX6AhWS4oUKHQCMBVgQ4dUDCA0&uact=5&oq=long+url+example&gs_lcp=Cgdnd3Mtd2l6EAMyBQgAEIAEMgUIABCGAzIFCAAQhgMyBQgAEIYDOgoIABBHENYEELADOgcIABCwAxBDOgQIABBDOgYIABAWEB5KBAhBGABKBAhGGABQmQJY5Qtg5AxoAXABeACAAaQBiAGfCJIBAzAuN5gBAKABAcgBCsABAQ&sclient=gws-wiz", "input": "https://www.google.com/search?q=long+url+example&hl=en&rls=en&ei=vmBMY5qoK5LFlwSAmJbABQ&ved=0ahUKEwia_9f5wuX6AhWS4oUKHQCMBVgQ4dUDCA0&uact=5&oq=long+url+example&gs_lcp=Cgdnd3Mtd2l6EAMyBQgAEIAEMgUIABCGAzIFCAAQhgMyBQgAEIYDOgoIABBHENYEELADOgcIABCwAxBDOgQIABBDOgYIABAWEB5KBAhBGABKBAhGGABQmQJY5Qtg5AxoAXABeACAAaQBiAGfCJIBAzAuN5gBAKABAcgBCsABAQ&sclient=gws-wiz", "title": "long url example - Google Search", "scheme": "https", "webserver": "gws", "content_type": "text/html", "method": "GET", "host": "142.250.114.104", "path": "/search?q=long+url+example&hl=en&rls=en&ei=vmBMY5qoK5LFlwSAmJbABQ&ved=0ahUKEwia_9f5wuX6AhWS4oUKHQCMBVgQ4dUDCA0&uact=5&oq=long+url+example&gs_lcp=Cgdnd3Mtd2l6EAMyBQgAEIAEMgUIABCGAzIFCAAQhgMyBQgAEIYDOgoIABBHENYEELADOgcIABCwAxBDOgQIABBDOgYIABAWEB5KBAhBGABKBAhGGABQmQJY5Qtg5AxoAXABeACAAaQBiAGfCJIBAzAuN5gBAKABAcgBCsABAQ&sclient=gws-wiz", "time": "1.08023725s", "a": [ "142.250.114.104", "142.250.114.105", "142.250.114.106", "142.250.114.147", "142.250.114.99", "142.250.114.103", "2607:f8b0:4023:1002::68", "2607:f8b0:4023:1002::69", "2607:f8b0:4023:1002::63", "2607:f8b0:4023:1002::67" ], "words": 14894, "lines": 248, "status_code": 200, "content_length": 499650, "failed": false, "stored_response_path": "**temp/store/fa3692d347cee373c38cdea83722f264.txt**" }