diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..3af2b9d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +# Configuration file for EditorConfig +# More information is available under http://EditorConfig.org + +# Ignore any other files further up in the file system +root = true + +# Configuration for all files +[*] +# Enforce Unix style line endings (\n only) +end_of_line = lf +# Always end files with a blank line +insert_final_newline = true +# Force space characters for indentation +indent_style = space +# Always indent by 4 characters +indent_size = 4 +# Remove whitespace characters at the end of line +trim_trailing_whitespace = true diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100755 index 0000000..219ce98 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,86 @@ +{ + "editor.tabSize": 4, + "editor.detectIndentation": false, + // "debug.toolBarLocation": "docked", + "editor.suggestSelection": "first", + "editor.fontSize": 14, + "editor.formatOnSave": true, + "editor.formatOnType": true, + "editor.wordWrapColumn": 250, + "editor.columnSelection": false, + "editor.mouseWheelZoom": true, + "explorer.confirmDelete": false, + "editor.renderWhitespace": "all", + "editor.renderControlCharacters": true, + "editor.suggest.shareSuggestSelections": true, + "editor.renderIndentGuides": false, + "debug.console.fontSize": 14, + "git.autofetch": true, + "[json]": { + "editor.defaultFormatter": "HookyQR.beautify" + }, + "[jsonc]": { + "editor.defaultFormatter": "HookyQR.beautify" + }, + "terminal.integrated.fontSize": 14, + "markdown.extension.toc.githubCompatibility": true, + "markdownlint.config": { + "MD033": false, + "MD034": false, + }, + "markdown.extension.katex.macros": {}, + "markdown.extension.toc.levels": "2..6", + "terminal.integrated.copyOnSelection": true, + // "files.autoSave": "afterDelay", + "files.insertFinalNewline": true, + "files.autoGuessEncoding": true, + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/*.pyc": true + }, + "python.analysis.extraPaths": [ + "./src", + "./mock", + "./test", + "./venv2/bin", + "./venv2/Lib/python2.7/site-packages" + ], + "python.analysis.logLevel": "Trace", + "python.analysis.watchSearchPaths": true, + "python.autoComplete.extraPaths": [ + "./src", + "./mock", + "./test", + "./venv2/bin", + "./venv2/Lib/python2.7/site-packages", + // "./venv/bin", + // "./venv/Lib/python3.7/site-packages", + ], + "python.pythonPath": "venv2/bin/python", + "python.formatting.provider": "yapf", + "python.languageServer": "Microsoft", + // "python.languageServer": "Pylance", + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=250"], + "python.testing.unittestArgs": [ + "-v", + "-s", + "./test", + "-p", + "test_*.py" + ], + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "python.testing.cwd": "${workspaceFolder}", + "python.testing.nosetestsEnabled": false, + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true, + "workbench.iconTheme": "vscode-icons", + "workbench.editor.enablePreview": true, + "workbench.editor.enablePreviewFromQuickOpen": true, + "window.zoomLevel": 0, +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..a90f8f8 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ + +# Python-DeMo + +Python DeMo + +- [一. 构建`venv`环境](#一-构建venv环境) + - [1.1 venv2](#11-venv2) + - [1.1.1 venv2 for mac](#111-venv2-for-mac) + - [1.1.2 venv2 for windows](#112-venv2-for-windows) + - [1.2 venv3](#12-venv3) + - [1.2.1 venv3 for mac](#121-venv3-for-mac) + - [1.2.2 venv3 for windows](#122-venv3-for-windows) +- [二. 多模块配置](#二-多模块配置) + +## 一. 构建`venv`环境 + +### 1.1 venv2 + +#### 1.1.1 venv2 for mac + +```bash +virtualenv -p /usr/local/bin/python2 --no-site-packages venv2 +``` + +#### 1.1.2 venv2 for windows + +```bash +python2 -m virtualenv -p C:\Python\Python27\python2.exe venv2 + +或 + +python2 -m virtualenv -p C:\Python\Python27\python2.exe venv2 > venv2.log +``` + +### 1.2 venv3 + +#### 1.2.1 venv3 for mac + +```bash +virtualenv -p /usr/local/bin/python3 --no-site-packages venv +``` + +#### 1.2.2 venv3 for windows + +```bash +python3 -m virtualenv -p C:\Python\Python38\python3.exe venv + +或 + +python3 -m virtualenv -p C:\Python\Python38\python3.exe venv > venv.log +``` + +## 二. 多模块配置 + +具体步骤: + +1. 在`venv2`和`venv` 下新建一个`python.pth`文件，具体目录为`venv2/lib/python2.x/site-packages/python.pth`和`venv/lib/python3.x/site-packages/python.pth`; +2. 在上述`python.pth`文件中写入多模块的`绝对路径`(自己设备上实际目录); + +```python +# venv2 环境 +# /Users/.../Python-DeMo/venv2/lib/python2.x/site-packages/python.pth +# +# venv 环境 +# /Users/.../Python-DeMo/venv/lib/python3.x/site-packages/python.pth + +/Users/.../Python-DeMo/src +/Users/.../Python-DeMo/test +/Users/.../Python-DeMo/mock +/Users/.../Python-DeMo/case +``` diff --git a/src/Demo.py b/case/Demo.py similarity index 100% rename from src/Demo.py rename to case/Demo.py diff --git a/src/annotation.py b/case/annotation.py similarity index 100% rename from src/annotation.py rename to case/annotation.py diff --git a/case/argparse/__init__.py b/case/argparse/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/argparse/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/argparse/test_argparse.py b/case/argparse/test_argparse.py similarity index 100% rename from src/argparse/test_argparse.py rename to case/argparse/test_argparse.py diff --git a/case/class/__init__.py b/case/class/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/class/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/class/test_class_2.7.py b/case/class/test_class_2.7.py similarity index 100% rename from src/class/test_class_2.7.py rename to case/class/test_class_2.7.py diff --git a/src/class/test_class_2.x.py b/case/class/test_class_2.x.py similarity index 100% rename from src/class/test_class_2.x.py rename to case/class/test_class_2.x.py diff --git a/src/class/test_class_3.7.py b/case/class/test_class_3.7.py similarity index 100% rename from src/class/test_class_3.7.py rename to case/class/test_class_3.7.py diff --git a/src/class/test_class_3.x.py b/case/class/test_class_3.x.py similarity index 100% rename from src/class/test_class_3.x.py rename to case/class/test_class_3.x.py diff --git a/case/file/__init__.py b/case/file/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/file/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/file/test_file.py b/case/file/test_file.py similarity index 100% rename from src/file/test_file.py rename to case/file/test_file.py diff --git a/case/function/__init__.py b/case/function/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/function/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/case/function/test_dict_sort.py b/case/function/test_dict_sort.py new file mode 100644 index 0000000..35db356 --- /dev/null +++ b/case/function/test_dict_sort.py @@ -0,0 +1,123 @@ +# -*- coding:utf-8 -*- + +import collections +import json +# import pprint + +tags = { + 'a': 2, + 'v0.0.2': { + 'stamp': 1576053555, + 'date': '2019-12-11', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 22 + }, + 'bc': '32', + 'ac': {}, + 'v0.0.3': { + 'stamp': 1576053555, + 'date': '2019-12-21', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 23 + }, + 'v0.0.1': { + 'stamp': 1576053555, + 'date': '2019-12-1', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 21 + }, + 's': ['2020-01-09', '2020-01-10'] +} + + +def test_dict_sort_slowly(): + ''' 使用sorted 会生成新的list，导致性能下降，建议使用内置sort ''' + global tags + tags_tuple_list = sorted(tags.items()) + # tags_tuple_list = sorted(tags.items(), key=lambda element: element[1]) + print('\n') + print(tags_tuple_list) + tags.clear() + print("tags is %s" % tags) + for item in tags_tuple_list: + tags[item[0]] = item[1] + print("tags is %s" % tags) + + +def test_dict_sort(): + global tags + # print(tags.items()) + keys = tags.keys() + keys.sort() + # tuple_with_tags = [(key, tags[key]) for key in keys] + # print("%s\n%s" % (type(tuple_with_tags), tuple_with_tags)) + tags_with_sort = collections.OrderedDict() # 有序字典 + for key in keys: + # tags_with_sort.setdefault(key, tags[key]) + tags_with_sort[key] = tags[key] + tags = tags_with_sort + + # prettyPrinter = pprint.PrettyPrinter(indent=4) + # prettyPrinter.pprint(tags) + + json_list = json.dumps(tags, indent=4) + print(json_list) + + # print() + # print(tags) + + +def test_print(): + ''' + { + 'v0.0.1': + { + 'stamp': 1576053555, + 'date': '2019-12-1', + 'commits': 21, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'authors': + { + 'DovSnier': 3 + } + }, + 'v0.0.2': + { + 'stamp': 1576053555, + 'date': '2019-12-11', + 'commits': 22, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'authors': + { + 'DovSnier': 3 + } + }, + 'v0.0.3': + { + 'stamp': 1576053555, + 'date': '2019-12-21', + 'commits': 23, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'authors': + { + 'DovSnier': 3 + } + } + } + ''' + pass + + +if __name__ == "__main__": + '''主函数入口''' + test_dict_sort() + # test_dict_sort_slowly() diff --git a/case/function/test_map.py b/case/function/test_map.py new file mode 100644 index 0000000..54d7d15 --- /dev/null +++ b/case/function/test_map.py @@ -0,0 +1,51 @@ +# -*- coding:utf-8 -*- + + +def test_map(): + tags = { + 'v0.0.2': { + 'stamp': 1576053555, + 'date': '2019-12-11', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 22 + }, + 'v0.0.3': { + 'stamp': 1576053555, + 'date': '2019-12-21', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 23 + }, + 'v0.0.1': { + 'stamp': 1576053555, + 'date': '2019-12-1', + 'authors': { + 'DovSnier': 3 + }, + 'hash': 'b76da094137669d3eb3edc985391366bc2bb93b9', + 'commits': 21 + } + } + # lambda + list_o = map(lambda el: (el[1]['date'], el[0]), tags.items()) + + # function + # list_o = map(test_tuple, tags.items()) + + print list_o + tags_sorted_by_date_desc = map(lambda el: el[1], reversed(sorted(list_o))) + print tags_sorted_by_date_desc + + +def test_tuple(element): + return (element[1]['date'], element[0]) + + +if __name__ == "__main__": + '''主函数入口''' + test_map() diff --git a/src/https.py b/case/https.py similarity index 56% rename from src/https.py rename to case/https.py index 73169bf..46fff47 100644 --- a/src/https.py +++ b/case/https.py @@ -4,12 +4,15 @@ import urllib2 import os # import shutil +import logging +import time def requests(): req_url = 'http://www.baidu.com/' - print(req_url) + # print(req_url) + logging.info('this request url is %s' % req_url) response = urllib2.urlopen(req_url) headers = response.headers @@ -44,12 +47,22 @@ def requests(): if any(headers): header_of_log = os.path.join(req_dir, 'response.header.log') file = open(header_of_log, 'w') - file.write(str(headers.dict)) + value = str(headers.dict) + logging.debug('this request headers is %s' % value) + file.write(value) file.close() body_of_log = os.path.join('out/request', 'response.body.log') file = open(body_of_log, 'w') - file.write(str(response.read())) + response_to_string = str(response.read()) + # logging.debug('this is response is %s' % response_to_string) + # + # logging.debug('this is debug message.') + # logging.info('this is info message.') + # logging.warning('this is warning message.') + # logging.error('this is error message.') + # logging.critical('this is critical message.') + file.write(response_to_string) file.close() @@ -62,5 +75,23 @@ def removeTree(dir): os.rmdir(os.path.join(root, name)) +def generateFileName(): + 'the generate out file name' + project_dir = os.path.abspath('.') + out_dir = os.path.join(project_dir, 'out') + http_dir = os.path.join(out_dir, 'http') + if not os.path.exists(http_dir): + os.makedirs(http_dir) + file_name = 'http_%s.log' % int(time.time()) + return os.path.join(http_dir, file_name) + + if __name__ == "__main__": + logging.basicConfig( + filename=generateFileName(), + filemode='a', + format='[%(asctime)s][%(levelname)8s] --- %(message)s', + # format= + # '[%(asctime)s][%(levelname)8s][%(filename)s:%(lineno)s] --- %(message)s', + level=logging.DEBUG) requests() diff --git a/case/logging/__init__.py b/case/logging/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/logging/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/logging/test_loggins.py b/case/logging/test_loggins.py similarity index 100% rename from src/logging/test_loggins.py rename to case/logging/test_loggins.py diff --git a/src/logging/test_thread_loggins.py b/case/logging/test_thread_loggins.py similarity index 100% rename from src/logging/test_thread_loggins.py rename to case/logging/test_thread_loggins.py diff --git a/src/readExcel.py b/case/readExcel.py similarity index 100% rename from src/readExcel.py rename to case/readExcel.py diff --git a/src/srcFile.py b/case/srcFile.py similarity index 100% rename from src/srcFile.py rename to case/srcFile.py diff --git a/case/standard_library/__init__.py b/case/standard_library/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/case/standard_library/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/standard_library/test_argv.py b/case/standard_library/test_argv.py similarity index 100% rename from src/standard_library/test_argv.py rename to case/standard_library/test_argv.py diff --git a/src/standard_library/test_dict.py b/case/standard_library/test_dict.py similarity index 100% rename from src/standard_library/test_dict.py rename to case/standard_library/test_dict.py diff --git a/src/standard_library/test_getopts.py b/case/standard_library/test_getopts.py similarity index 100% rename from src/standard_library/test_getopts.py rename to case/standard_library/test_getopts.py diff --git a/src/standard_library/test_lock.py b/case/standard_library/test_lock.py similarity index 100% rename from src/standard_library/test_lock.py rename to case/standard_library/test_lock.py diff --git a/src/standard_library/test_os_fork.py b/case/standard_library/test_os_fork.py similarity index 100% rename from src/standard_library/test_os_fork.py rename to case/standard_library/test_os_fork.py diff --git a/src/standard_library/test_popen.py b/case/standard_library/test_popen.py similarity index 100% rename from src/standard_library/test_popen.py rename to case/standard_library/test_popen.py diff --git a/src/standard_library/test_popen_question.py b/case/standard_library/test_popen_question.py similarity index 100% rename from src/standard_library/test_popen_question.py rename to case/standard_library/test_popen_question.py diff --git a/src/standard_library/test_queue.py b/case/standard_library/test_queue.py similarity index 100% rename from src/standard_library/test_queue.py rename to case/standard_library/test_queue.py diff --git a/src/standard_library/test_send_smtp.py b/case/standard_library/test_send_smtp.py similarity index 100% rename from src/standard_library/test_send_smtp.py rename to case/standard_library/test_send_smtp.py diff --git a/src/standard_library/test_send_smtp_ssl.py b/case/standard_library/test_send_smtp_ssl.py similarity index 100% rename from src/standard_library/test_send_smtp_ssl.py rename to case/standard_library/test_send_smtp_ssl.py diff --git a/src/standard_library/test_sendmail.py b/case/standard_library/test_sendmail.py similarity index 100% rename from src/standard_library/test_sendmail.py rename to case/standard_library/test_sendmail.py diff --git a/src/standard_library/test_thread.py b/case/standard_library/test_thread.py similarity index 100% rename from src/standard_library/test_thread.py rename to case/standard_library/test_thread.py diff --git a/src/standard_library/test_threading.py b/case/standard_library/test_threading.py similarity index 100% rename from src/standard_library/test_threading.py rename to case/standard_library/test_threading.py diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..1026f19 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,35 @@ + +# 文档 + +- [一. 查看函数命令](#一-查看函数命令) + - [1.1 Mac/Linux 环境查看函数](#11-maclinux-环境查看函数) + - [1.2. Windows 环境查看函数](#12-windows-环境查看函数) + - [1.3. 阅读源码命令](#13-阅读源码命令) + +## 一. 查看函数命令 + +### 1.1 Mac/Linux 环境查看函数 + +```python +pydoc function // 具体参考函数 +``` + +### 1.2. Windows 环境查看函数 + +```python +python -m pydoc function // 具体参考函数 +``` + +### 1.3. 阅读源码命令 + +阅读命令： + +```python +python3 -m pydoc -p 8081 +``` + +自己封装了一个脚本，调用命令： + +```python +source ./doc/pydoc.py +``` diff --git a/doc/readMe.md b/doc/readMe.md deleted file mode 100644 index 2722fd6..0000000 --- a/doc/readMe.md +++ /dev/null @@ -1,29 +0,0 @@ - -## 查看函数命令 - -### 1. Mac/Linux 环境查看函数 - -``` -pydoc function // 具体参考函数 -``` - -### 2. Windows 环境查看函数 - -``` -python -m pydoc function // 具体参考函数 -``` - -### 3. 阅读源码命令 - -阅读命令： - -``` -python3 -m pydoc -p 8081 -``` - -自己封装了一个脚本，调用命令： - -``` -source ./doc/pydoc.py -``` - diff --git a/doc/source.md b/doc/source.md new file mode 100755 index 0000000..94ab81b --- /dev/null +++ b/doc/source.md @@ -0,0 +1,10 @@ +# 阅读 + +尝试去一些网站读源码: + +- https://github.com/ +- https://launchpad.net/ +- https://koders.com/ +- https://bitbucket.org/ +- https://www.python.org/dev/peps/ +- https://stackoverflow.com/ diff --git a/readMe.md b/readMe.md deleted file mode 100644 index bc47b12..0000000 --- a/readMe.md +++ /dev/null @@ -1,44 +0,0 @@ - -# Python-DeMo - -Python DeMo - - - -## 构建`venv`环境 - -### venv2 - -#### 1. venv2 for mac - -``` -virtualenv -p /usr/local/bin/python2 --no-site-packages venv2 -``` - -#### 2. venv2 for windows - -``` -python2 -m virtualenv -p C:\Python\Python27\python2.exe venv2 - -或 - -python2 -m virtualenv -p C:\Python\Python27\python2.exe venv2 > venv2.log -``` - -### venv3 - -#### 1. venv3 for mac - -``` -virtualenv -p /usr/local/bin/python3 --no-site-packages venv -``` - -#### 2. venv3 for windows - -``` -python3 -m virtualenv -p C:\Python\Python38\python3.exe venv - -或 - -python3 -m virtualenv -p C:\Python\Python38\python3.exe venv > venv.log -``` diff --git a/src/com/__init__.py b/src/com/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/com/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/com/dvsnier/__init__.py b/src/com/dvsnier/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/com/dvsnier/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/com/dvsnier/conf/__init__.py b/src/com/dvsnier/conf/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/com/dvsnier/conf/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/com/dvsnier/conf/common_conf.py b/src/com/dvsnier/conf/common_conf.py new file mode 100644 index 0000000..34801c8 --- /dev/null +++ b/src/com/dvsnier/conf/common_conf.py @@ -0,0 +1,26 @@ +# -*- coding:utf-8 -*- + +import logging +from com.dvsnier.dir.common_dir import generate_complex_file_name + + +def logging_conf(kwargs): + ''' + the logging conf info: + logging_conf(**kwargs={output_dir_name=\' \', file_name=\' \'}) + ''' + if kwargs.get('output_dir_name') is None or len( + kwargs.get('output_dir_name')) == 0: + raise KeyError('the current kwargs[output_dir_name] is empty.') + if kwargs.get('file_name') is None or len(kwargs.get('file_name')) == 0: + raise KeyError('the current kwargs[file_name] is empty.') + file_name = generate_complex_file_name(kwargs['output_dir_name'], + kwargs['file_name']) + logging.basicConfig( + filename=file_name, + filemode='a', + format='[%(asctime)s][%(levelname)8s] --- %(message)s', + # format= + # '[%(asctime)s][%(levelname)8s][%(filename)s:%(lineno)s] --- %(message)s', + level=logging.DEBUG) + logging.info('this current file is %s' % (file_name)) diff --git a/src/com/dvsnier/dir/__init__.py b/src/com/dvsnier/dir/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/com/dvsnier/dir/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/com/dvsnier/dir/common_dir.py b/src/com/dvsnier/dir/common_dir.py new file mode 100644 index 0000000..e6a79b9 --- /dev/null +++ b/src/com/dvsnier/dir/common_dir.py @@ -0,0 +1,46 @@ +# -*- coding:utf-8 -*- + +import os +# import logging +import time + + +def generate_complex_file_name(output_dir_name, file_name): + 'the generate complex file name' + output_dir = mk_output_dir(output_dir_name) + name = str("%s_%s.log" % (file_name, int(time.time()))) + return os.path.join(output_dir, name) + + +def generate_file_name(output_dir_name, file_name): + 'the generate out file name' + output_dir = mk_output_dir(output_dir_name) + return os.path.join(output_dir, file_name) + + +def mk_output_dir(output_dir_name): + 'the initialize output dir' + # root_dir = os.path.dirname(os.path.abspath('.')) + # logging.debug('the current root_dir is %s' % root_dir) + project_dir = os.path.abspath('.') + # logging.debug('the current project_dir is %s' % project_dir) + # src_dir = os.path.join(project_dir, 'src') + # logging.debug('the current src_dir is %s' % src_dir) + out_dir = os.path.join(project_dir, 'out') + # logging.debug('the current out_dir is %s' % out_dir) + output_dir = os.path.join(out_dir, output_dir_name) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + # logging.debug('the current output_dir is %s' % output_dir) + return output_dir + + +def mk_children_dir(output_dir_name, sub_dir_name): + 'the initialize children dir' + if not os.path.exists(output_dir_name): + os.makedirs(output_dir_name) + children_dir = os.path.join(output_dir_name, sub_dir_name) + if not os.path.exists(children_dir): + os.makedirs(children_dir) + # logging.debug('the current children_dir is %s' % children_dir) + return children_dir diff --git a/src/com/dvsnier/http/__init__.py b/src/com/dvsnier/http/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/com/dvsnier/http/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/com/dvsnier/http/https.py b/src/com/dvsnier/http/https.py new file mode 100644 index 0000000..8c52dd0 --- /dev/null +++ b/src/com/dvsnier/http/https.py @@ -0,0 +1,46 @@ +# -*- coding:utf-8 -*- + +from com.dvsnier.conf.common_conf import logging_conf +import urllib2 +import logging + + +class Request(object): + 'the request class' + + _url = None # the request url address + + # def __init__(self): + # super(Request, self).__init__() + # kwargs = {'output_dir_name': 'request', 'file_name': 'request'} + # logging_conf(kwargs) + + def __init__(self, *url): + super(Request, self).__init__() + self._url = url + kwargs = {'output_dir_name': 'request', 'file_name': 'request'} + logging_conf(kwargs) + + def requests(self, *req_url): + 'the request address' + if len(req_url) == 0 or req_url is None: + req_url = self._url + if len(req_url) == 0: + return + logging.info('this request url is %s' % req_url) + response = urllib2.urlopen(req_url[0]) + headers = response.headers + if any(headers): + logging.debug('this request headers is %s' % str(headers.dict)) + + # response_to_string = str(response.read()) + # logging.debug('this is response is %s' % response_to_string) + return response + + def getUrl(self): + 'the get url' + if not len(self._url) == 0: + logging.info('the current url is %s' % self._url) + else: + logging.info('the current url is empty') + return self._url diff --git a/src/demo/__init__.py b/src/demo/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/src/demo/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/src/demo/http_case.py b/src/demo/http_case.py new file mode 100644 index 0000000..775ebd6 --- /dev/null +++ b/src/demo/http_case.py @@ -0,0 +1,20 @@ +# -*- coding:utf-8 -*- + +from com.dvsnier.http.https import Request +from com.dvsnier.dir.common_dir import generate_complex_file_name + +_request = None + + +def run(url): + return _request.requests(url) + + +if __name__ == "__main__": + _request = Request() + url = 'http://www.xuexi.la/sudu/523441.html' + # url = 'https://www.thepaper.cn/newsDetail_forward_9864719' + response = run(url) + response_file = generate_complex_file_name('request', 'response') + with open(response_file, 'w+') as f: + f.write(str(response.read())) diff --git a/template/__init__.py b/template/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/template/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/template/template_test.py b/template/test_template.py similarity index 100% rename from template/template_test.py rename to template/test_template.py diff --git a/test/com/__init__.py b/test/com/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/test/com/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/test/com/dvsnier/__init__.py b/test/com/dvsnier/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/test/com/dvsnier/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/test/com/dvsnier/conf/__init__.py b/test/com/dvsnier/conf/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/test/com/dvsnier/conf/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/test/com/dvsnier/conf/test_common_conf.py b/test/com/dvsnier/conf/test_common_conf.py new file mode 100644 index 0000000..2080409 --- /dev/null +++ b/test/com/dvsnier/conf/test_common_conf.py @@ -0,0 +1,37 @@ +# -*- coding:utf-8 -*- + +import unittest +from com.dvsnier.conf.common_conf import logging_conf + + +class Test_Common_Conf(unittest.TestCase): + ''' the test common conf ''' + @classmethod + def setUpClass(cls): + print("...the set up...") + print + + def setUp(self): + return super(Test_Common_Conf, self).setUp() + + def test_logging_conf(self): + 'the test logging conf' + # kwargs = {'output_dir_name': ' ', 'file_name': ' '} + # kwargs = {'output_dir_name': 'request', 'file_name': ' '} + # kwargs = {'output_dir_name': ' ', 'file_name': 'request'} + kwargs = {'output_dir_name': 'request', 'file_name': 'request'} + logging_conf(kwargs) + print "the test test_logging_conf is succeed." + pass + + def tearDown(self): + return super(Test_Common_Conf, self).tearDown() + + @classmethod + def tearDownClass(cls): + print + print("...the tear down...") + + +if __name__ == '__main__': + unittest.main() diff --git a/test/com/dvsnier/dir/__init__.py b/test/com/dvsnier/dir/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/test/com/dvsnier/dir/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/test/com/dvsnier/dir/test_common_dir.py b/test/com/dvsnier/dir/test_common_dir.py new file mode 100644 index 0000000..6434972 --- /dev/null +++ b/test/com/dvsnier/dir/test_common_dir.py @@ -0,0 +1,53 @@ +# -*- coding:utf-8 -*- + +import unittest + +from com.dvsnier.dir.common_dir import generate_complex_file_name, generate_file_name, mk_children_dir, mk_output_dir + + +class Test_Common_Dir(unittest.TestCase): + ''' the test common dir ''' + @classmethod + def setUpClass(cls): + print("...the set up...") + + def setUp(self): + return super(Test_Common_Dir, self).setUp() + + def test_generate_complex_file_name(self): + dir_name = 'https' + file_name = 'test_name' + output = generate_complex_file_name(dir_name, file_name) + print("\nthe test generate_complex_file_name(%s) is succeed." % output) + self.assertIsNotNone(output, + 'test_generate_complex_file_name is error.') + + def test_generate_file_name(self): + dir_name = 'https' + file_name = 'test_name' + output = generate_file_name(dir_name, file_name) + print("\nthe test test_generate_file_name(%s) is succeed." % output) + self.assertIsNotNone(output, 'test_generate_file_name is error.') + + def test_mk_output_dir(self): + dir_name = 'https' + output = mk_output_dir(dir_name) + print("\nthe test mk_output_dir(%s) is succeed." % output) + self.assertIsNotNone(output, 'test_mk_output_dir is error.') + + def test_mk_children_dir(self): + sub_name = 'children' + output = mk_children_dir('out/http', sub_dir_name=sub_name) + print("\nthe test mk_children_dir(%s) is succeed." % output) + self.assertIsNotNone(output, 'test_mk_children_dir is error.') + + def tearDown(self): + return super(Test_Common_Dir, self).tearDown() + + @classmethod + def tearDownClass(cls): + print("...the tear down...") + + +if __name__ == '__main__': + unittest.main() diff --git a/test/com/dvsnier/http/__init__.py b/test/com/dvsnier/http/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/test/com/dvsnier/http/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/test/com/dvsnier/http/test_https.py b/test/com/dvsnier/http/test_https.py new file mode 100644 index 0000000..69b230e --- /dev/null +++ b/test/com/dvsnier/http/test_https.py @@ -0,0 +1,67 @@ +# -*- coding:utf-8 -*- + +import unittest +from com.dvsnier.http.https import Request + + +def test_two(): + # the origin request object, then test case two + # _request = Request(url) + # _request.requests() + pass + + +class Test_Https(unittest.TestCase): + ''' the test https ''' + @classmethod + def setUpClass(cls): + print("...the set up...") + print + + def setUp(self): + return super(Test_Https, self).setUp() + + def test_requests_one(self): + 'the test requests one' + # url = 'http://www.xuexi.la/sudu/523441.html' + _request = Request() + self.assertIsNotNone(_request, 'test_requests_one is error.') + self.assertIsNone(_request.requests()) + + def test_requests_two(self): + 'the test requests two' + url = 'http://www.xuexi.la/sudu/523441.html' + _request = Request(url) + self.assertIsNotNone(_request, 'test_requests_two is error') + response = _request.requests() + self.assertIsNotNone(response, 'test_requests_two is error.') + + def test_requests_three(self): + 'the test requests three' + url = 'http://www.xuexi.la/sudu/523441.html' + _request = Request() + self.assertIsNotNone(_request, 'test_requests_three is error') + response = _request.requests(url) + self.assertIsNotNone(_request.getUrl(), + 'test_requests_three is error.') + self.assertIsNotNone(response, 'test_requests_three is error.') + + def test_getUrl(self): + 'the test get url' + url = 'http://www.xuexi.la/sudu/523441.html' + _request = Request(url) + self.assertIsNotNone(_request.getUrl(), 'test_getUrl is error.') + _request = Request() + self.assertTupleEqual(_request.getUrl(), (), 'test_getUrl is error.') + + def tearDown(self): + return super(Test_Https, self).tearDown() + + @classmethod + def tearDownClass(cls): + print + print("...the tear down...") + + +if __name__ == '__main__': + unittest.main()