[+] 加入配置文件，扫描可自定义相关参数

README.MD

@@ -25,6 +25,7 @@
 - 本人并不崇尚命令行哲学，对于这个是能简则简
 - 直接使用`python w9scan.py`即可使用向导模式使用w9scan
 - 向导模式中输入URL时可以输入类似`@1.txt`,代表批量扫描1.txt内的URL
+- 有关配置线程，超时时间，header头等等参数可以打开config.conf配置
 
 ## Features
 

UPDATELOG.MD

@@ -1,7 +1,11 @@
 ## 升级记录日志
 
-- 接下来要做的:  
-    - [ ] 最近看了POC-T的代码颇有感悟，学习poc-t的代码
+- 接下来要做的: 
+    - [ ] 爬虫模块重新优化
+- TODO
+    - [] 识别网页稳定性动态调整线程
+    - [] 识别网站跳转后的地址
+- 1.8.8 加入读取配置文件
 - 1.8.7 插件如果连接超时将加入重试队列中重试一次
 - 1.8.6 参考[WAScan](https://github.com/m4ll0k/WAScan) 1.增加phpinfo泄露路径  2.增加框架识别功能 3.header cookie报告 4.检测Cross Site Tracing
 - 1.8.5 增加debug命令系统，增加日志记录系统,增加了随机banner（参见s7san）

lib/core/common.py

@@ -39,6 +39,40 @@ def isListLike(value):
 
     return isinstance(value, (list, tuple, set))
 
+def flattenValue(value):
+    """
+    Returns an iterator representing flat representation of a given value
+
+    >>> [_ for _ in flattenValue([[u'1'], [[u'2'], u'3']])]
+    [u'1', u'2', u'3']
+    """
+
+    for i in iter(value):
+        if isListLike(i):
+            for j in flattenValue(i):
+                yield j
+        else:
+            yield i
+
+def unArrayizeValue(value):
+    """
+    Makes a value out of iterable if it is a list or tuple itself
+
+    >>> unArrayizeValue([u'1'])
+    u'1'
+    """
+
+    if isListLike(value):
+        if not value:
+            value = None
+        elif len(value) == 1 and not isListLike(value[0]):
+            value = value[0]
+        else:
+            _ = filter(lambda _: _ is not None, (_ for _ in flattenValue(value)))
+            value = _[0] if len(_) > 0 else None
+
+    return value
+
 def getUnicode(value, encoding=None, noneToNull=False):
     """
     Return the unicode representation of the supplied value:

lib/core/engine.py

@@ -14,8 +14,6 @@ def pluginScan():
         return False
     urlconfig.scanport = False
     urlconfig.find_service = False
-    urlconfig.threadNum = 5
-    urlconfig.deepMax = 100
     urlconfig.diyPlugin = LIST_PLUGINS
     startTime = time.clock()
     e = Exploit_run(urlconfig.threadNum)

lib/core/enums.py

@@ -11,4 +11,10 @@ class CUSTOM_LOGGING:
 class EXIT_STATUS:
     SYSETM_EXIT = 0
     ERROR_EXIT = 1
-    USER_QUIT = 2
+    USER_QUIT = 2
+
+class OPTION_TYPE:
+    BOOLEAN = "boolean"
+    INTEGER = "integer"
+    FLOAT = "float"
+    STRING = "string"

lib/core/exploit.py

@@ -6,7 +6,7 @@
 from lib.core.data import paths
 from thirdparty import miniCurl
 from lib.utils import until
-from lib.core.data import urlconfig,logger
+from lib.core.data import urlconfig,logger,w9config
 from thirdparty import hackhttp
 from thirdparty.ThreadPool import w8_threadpool
 from lib.core.common import printMessage
@@ -117,8 +117,14 @@ def load_modules(self,service,url):
                 pluginObj.debug = self._debug
                 pluginObj.util = until
                 pluginObj._G = self._TargetScanAnge
-                pluginObj.hackhttp = hackhttp.hackhttp()
                 pluginObj.ThreadPool = w8_threadpool
+
+                if w9config.TimeOut is None:
+                    w9config.TimeOut = 10
+                if w9config.Cookie is None:
+                    w9config.Cookie = ""
+                conpool = hackhttp.httpconpool(20,timeout=w9config.TimeOut)
+                pluginObj.hackhttp = hackhttp.hackhttp(conpool=conpool,cookie_str=w9config.Cookie,user_agent = w9config.UserAgent,headers=w9config.headers)
 
                 pluginObj_tuple = pluginObj.assign(service, url)
                 if not isinstance(pluginObj_tuple, tuple):  # 判断是否是元组

lib/core/option.py

@@ -4,7 +4,7 @@
 # author = w8ay
 from lib.core.update import updateProgram
 import sys,os,logging,time
-from lib.core.data import paths,logger,urlconfig
+from lib.core.data import paths,logger,urlconfig,w9config
 from lib.core.common import makeurl,printMessage
 from lib.core.settings import LIST_PLUGINS
 from lib.core.enums import CUSTOM_LOGGING
@@ -19,6 +19,17 @@ def initOption(args):
     urlconfig.search = False
     urlconfig.usePlugin = False
 
+    # 这里初始化配置文件的选项
+    urlconfig.threadNum = w9config.thread
+    if urlconfig.threadNum is None:
+        urlconfig.threadNum = 5
+    urlconfig.deepMax = w9config.crawlerDeep
+    if urlconfig.deepMax is None:
+        urlconfig.deepMax = 100
+
+    urlconfig.threadNum = int(urlconfig.threadNum)
+    urlconfig.deepMax = int(urlconfig.deepMax)
+
     bannerOutput(args)
     setLoggingLevel(args)
     checkUpdate(args)  # 检测更新
@@ -121,13 +132,4 @@ def guideRegister(args):
         urlconfig.find_service = True
         input_scanport = raw_input('[2.1] Need you scan all ports ?(Y/N) (default N)> ')
         if input_scanport.lower() in ("y","yes"):
-            urlconfig.scanport = True
-
-    urlconfig.threadNum = raw_input('[3] You need start number of thread (default 5) > ')
-    if urlconfig.threadNum == '':
-        urlconfig.threadNum = 5
-
-    urlconfig.threadNum = int(urlconfig.threadNum)
-    urlconfig.deepMax = raw_input('[4] Set the depth of the crawler (default 100 | 0 don\'t use crawler ) > ')
-    if urlconfig.deepMax == '':
-        urlconfig.deepMax = 100
+            urlconfig.scanport = True

lib/core/settings.py

@@ -3,7 +3,7 @@
 import subprocess
 
 # w9scan version
-VERSION = "1.8.7"
+VERSION = "1.8.8"
 Site = "https://github.com/boy-hack/w9scan"
 AUTHOR = "w8ay"
 MAIL = "master@hacking8.com"

lib/utils/configfile.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+import os
+import codecs
+from lib.core.data import logger
+from ConfigParser import RawConfigParser,DEFAULTSECT
+from lib.core.exception import ToolkitSystemException
+from lib.core.common import getUnicode,unArrayizeValue
+from lib.core.enums import OPTION_TYPE
+from lib.core.data import w9config
+
+config = None
+
+optDict = {
+    "Config":{
+        "thread":            "integer",
+        "crawlerDeep":               "integer",
+        "TimeOut":           "integer",
+        "UserAgent":          "string",
+        "Cookie":       "string",
+        "headers":       "string"
+    }
+}                             
+
+def configFileProxy(section, option, datatype):
+    """
+    Parse configuration file and save settings into the configuration
+    advanced dictionary.
+    """
+
+    global config
+
+    if config.has_option(section, option):
+        try:
+            if datatype == OPTION_TYPE.BOOLEAN:
+                value = config.getboolean(section, option) if config.get(section, option) else False
+            elif datatype == OPTION_TYPE.INTEGER:
+                value = config.getint(section, option) if config.get(section, option) else 0
+            elif datatype == OPTION_TYPE.FLOAT:
+                value = config.getfloat(section, option) if config.get(section, option) else 0.0
+            else:
+                value = config.get(section, option)
+        except ValueError, ex:
+            errMsg = "error occurred while processing the option "
+            errMsg += "'%s' in provided configuration file ('%s')" % (option, getUnicode(ex))
+            raise ToolkitSystemException(errMsg)
+
+        if value:
+            w9config[option] = value
+        else:
+            w9config[option] = None
+    else:
+        debugMsg = "missing requested option '%s' (section " % option
+        debugMsg += "'%s') into the configuration file, " % section
+        debugMsg += "ignoring. Skipping to next."
+        logger.debug(debugMsg)
+
+def openFile(filename, mode='r', encoding="utf8", errors="replace", buffering=1):  # "buffering=1" means line buffered (Reference: http://stackoverflow.com/a/3168436)
+    """
+    Returns file handle of a given filename
+    """
+
+    try:
+        return codecs.open(filename, mode, encoding, errors, buffering)
+    except IOError:
+        errMsg = "there has been a file opening error for filename '%s'. " % filename
+        errMsg += "Please check %s permissions on a file " % ("write" if \
+          mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
+        errMsg += "and that it's not locked by another process."
+        raise ToolkitSystemException(errMsg)
+
+def checkFile(filename, raiseOnError=True):
+    """
+    Checks for file existence and readability
+    """
+
+    valid = True
+
+    try:
+        if filename is None or not os.path.isfile(filename):
+            valid = False
+    except UnicodeError:
+        valid = False
+
+    if valid:
+        try:
+            with open(filename, "rb"):
+                pass
+        except:
+            valid = False
+
+    if not valid and raiseOnError:
+        raise ToolkitSystemException("unable to read file '%s'" % filename)
+
+    return valid
+
+class UnicodeRawConfigParser(RawConfigParser):
+    """
+    RawConfigParser with unicode writing support
+    """
+
+    def write(self, fp):
+        """
+        Write an .ini-format representation of the configuration state.
+        """
+
+        if self._defaults:
+            fp.write("[%s]\n" % DEFAULTSECT)
+
+            for (key, value) in self._defaults.items():
+                fp.write("%s = %s\n" % (key, getUnicode(value, "UTF8").replace('\n', '\n\t')))
+
+            fp.write("\n")
+
+        for section in self._sections:
+            fp.write("[%s]\n" % section)
+
+            for (key, value) in self._sections[section].items():
+                if key != "__name__":
+                    if value is None:
+                        fp.write("%s\n" % (key))
+                    else:
+                        fp.write("%s = %s\n" % (key, getUnicode(value, "UTF8").replace('\n', '\n\t')))
+
+            fp.write("\n")
+
+def configFileParser(configFile):
+    """
+    Parse configuration file and save settings into the configuration
+    advanced dictionary.
+    """
+
+    global config
+
+    debugMsg = "parsing configuration file"
+    logger.debug(debugMsg)
+
+    checkFile(configFile)
+    configFP = openFile(configFile, "rb")
+
+    try:
+        config = UnicodeRawConfigParser()
+        config.readfp(configFP)
+    except Exception, ex:
+        errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % ex
+        raise ToolkitSystemException(errMsg)
+
+    if not config.has_section("Config"):
+        errMsg = "missing a mandatory section 'Config' in the configuration file"
+        raise ToolkitSystemException(errMsg)
+
+    mandatory = False
+
+    for option in ("thread","crawlerDeep","TimeOut","UserAgent","Cookie","headers"):
+        if config.has_option("Config", option):
+            mandatory = True
+            break
+
+    if not mandatory:
+        errMsg = "missing a mandatory option in the configuration file "
+        errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile, sitemapUrl or wizard)"
+        raise ToolkitSystemException(errMsg)
+
+    for family, optionData in optDict.items():
+        for option, datatype in optionData.items():
+            datatype = unArrayizeValue(datatype)
+            configFileProxy(family, option, datatype)

thirdparty/hackhttp.py

@@ -151,12 +151,14 @@ def _release_connect(self, url):
 
 class hackhttp():
 
-    def __init__(self, conpool=None, cookie_str=None, throw_exception=True):
+    def __init__(self, conpool=None, cookie_str=None, throw_exception=True,user_agent = None,headers=None):
         """conpool: 创建的连接池最大数量，类型为 int，默认为 10
 
             cookie_str: 用户自己定义的 Cookie，类型为 String
 
             throw_exception: 是否抛出遇到的异常，类型为 bool，默认为 True
+
+            为了全局配置useragent header 这里加入相关选项
         """
         self.throw_exception = throw_exception
         if conpool is None:
@@ -173,6 +175,8 @@ def __init__(self, conpool=None, cookie_str=None, throw_exception=True):
                     cookiekey, cookievalue = cookiepart.split("=", 1)
                     self.initcookie[cookiekey.strip()] = cookievalue.strip()
         self.cookiepool = {}
+        self.user_agent = user_agent
+        self.headers = headers
 
     def _get_urlinfo(self, url):
         p = urlparse.urlparse(url)
@@ -331,6 +335,12 @@ def http(self, url, post=None, **kwargs):
         location = kwargs.get('location', True)
         throw_exception = kwargs.get('throw_exception', self.throw_exception)
 
+        if not headers and self.headers:
+            headers = self.headers
+
+        if self.user_agent is not None and kwargs.get('user_agent', None) is None:
+            kwargs["user_agent"] = self.user_agent
+
         if headers and (isinstance(headers, str) or isinstance(headers, unicode)):
             headers = httpheader(StringIO.StringIO(headers), 0).dict
         for arg_key, h in[

w9scan.py

@@ -22,6 +22,7 @@
 from lib.core.exploit import Exploit_run
 from lib.core.option import initOption
 from lib.core.settings import IS_WIN, LIST_PLUGINS, VERSION
+from lib.utils.configfile import configFileParser
 from thirdparty.colorama.initialise import init as winowsColorInit
 
 sys.dont_write_bytecode = True  # 不生成pyc文件
@@ -83,7 +84,7 @@ def main():
     if IS_WIN:
         winowsColorInit()
     Banner()
-
+    configFileParser("config.conf")
 
     try:
         initOption(args)