Merge pull request qd-today#116 from a76yyyy/master

同步 a76yyyy/qiandao

.github/workflows/Build Image.yml

@@ -4,16 +4,21 @@ jobs:
   build:
       runs-on: ubuntu-latest
       name: Build image job
+
       steps:
           - name: Checkout master
             uses: actions/checkout@master
+          - name: Get version
+            id: get_version
+            if: startsWith(github.ref, 'refs/tags/')
+            run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
           - name: Build and publish image
             uses: ilteoood/docker_buildx@master
             if: startsWith(github.ref, 'refs/tags/')
             with:
               publish: true
               imageName: a76yyyy/qiandao # dockerid/imageName
-              platform: linux/arm64,linux/amd64,linux/arm/v7 # 你准备构建的镜像平台
-              tag: latest
+              platform: linux/386,linux/arm64,linux/amd64,linux/arm/v6,linux/arm/v7 # 你准备构建的镜像平台
+              tag: latest,${{ steps.get_version.outputs.VERSION }}
               dockerUser: ${{ secrets.DOCKER_USERNAME }} # docker hub userid 在setting创建secrets name=DOCKER_USERNAME  value=dockerid
               dockerPassword: ${{ secrets.DOCKER_PASSWORD }} # docker hub password,在setting创建secrets name=DOCKER_PASSWORD  value=dockerpassword

Dockerfile

@@ -21,13 +21,36 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositorie
 
 # Install packages
 RUN apk update \
-    && apk add --no-cache openrc redis libcurl bash git autoconf g++ tzdata nano openssh-client
+    && apk add --no-cache openrc redis bash git autoconf g++ tzdata nano openssh-client
 
 # Needed for pycurl
 ENV PYCURL_SSL_LIBRARY=openssl
+ENV CURL_VERSION 7.78.0
 
-# Install packages only needed for building, install and clean on a single layer
-RUN apk add --no-cache --virtual .build-dependencies build-base curl-dev 
+# For nghttp2-dev, we need this respository.
+RUN echo https://dl-cdn.alpinelinux.org/alpine/edge/testing >>/etc/apk/repositories 
+
+RUN apk add --update --no-cache openssl openssl-dev nghttp2-dev ca-certificates zlib zlib-dev brotli brotli-dev zstd zstd-dev
+RUN apk add --update --no-cache --virtual curldeps make perl && \
+wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.bz2 && \
+tar xjvf curl-$CURL_VERSION.tar.bz2 && \
+rm curl-$CURL_VERSION.tar.bz2 && \
+cd curl-$CURL_VERSION && \
+./configure \
+    --with-nghttp2=/usr \
+    --prefix=/usr \
+    --with-ssl \
+    --enable-ipv6 \
+    --enable-unix-sockets \
+    --without-libidn \
+    --disable-static \
+    --disable-ldap \
+    --with-pic && \
+make && \
+make install && \
+cd .. && \
+rm -r curl-$CURL_VERSION && \
+apk del curldeps
 
 # Setting openrc-redis
 RUN rc-status -a \
@@ -48,8 +71,8 @@ RUN mkdir -p /root/.ssh \
 # Pip install modules
 RUN pip install --upgrade setuptools \
     && pip install --no-cache-dir -r requirements.txt \
-    && apk del .build-dependencies \
-    && rm -rf /var/cache/apk/* 
+    && rm -rf /var/cache/apk/* \
+    && rm -rf /usr/share/man/* 
 
 ENV PORT 80
 EXPOSE $PORT/tcp

Pipfile

@@ -15,11 +15,12 @@ pytz = "*"
 croniter = "*"
 Jinja2 = "*"
 umsgpack = "*"
-pycurl = "<7.43.0.6"
 redis = "*"
 mysql-connector-python = "*"
+incremental = "*"
 
 [dev-packages]
+pycurl = "<7.43.0.6"
 
 [requires]
 python_version = "3.8"

README.md

@@ -6,7 +6,9 @@
 
 [HAR editor 使用指南](docs/har-howto.md)
 
-**<big>操作前请一定要记得备份数据库</big>**
+**操作前请一定要记得备份数据库**
+
+**不要同时开启新旧版签到框架，或将不同运行中容器的数据库映射为同一文件**
 
 使用Docker部署站点
 ==========
@@ -115,6 +117,34 @@ mailgun_key = ""
 
 更新日志
 =========
+## 2021.09.05 更新
+1. 优化任务前值显示方案
+2. 修复IPv6访问时的bug
+3. 管理用户界面添加用户最后登录IP显示
+4. 我的页面添加分组任务“全选/反选”复选框
+## 2021.09.03 更新
+1. 修复更新新版框架后因旧版框架cookie缓存导致的500错误
+2. 修复util编解码问题
+3. 添加368和armv6架构容器构建
+4. 优化插入请求功能
+5. 修复pycurl导致若干500和599错误
+6. 更新需求模块
+
+**Docker已预装Curl环境，默认不安装pycurl模组**
+
+```
+# 如需使用Proxy功能请安装PyCurl
+# Windows源码运行, 请执行 pip install pycurl==7.43.0.5 
+pip install pycurl # pip3 install pycurl
+```
+
+## 2021.09.02 更新
+1. 修复Image解码失败的bug
+2. 添加请求时限设置
+3. 修复延时API超过请求时限导致的bug
+4. 查看任务的模板数据时自动导入任务变量(by [billypon](https://github.com/billypon/qiandao))
+5. 默认去除pycurl模组，解决部分500和599错误
+
 ## 2021.08.07 更新
 1. 更新Wiki
 2. 修复网页编码导致的Body解析bug
@@ -128,7 +158,6 @@ mailgun_key = ""
 6. 修复delay延时功能
 7. 增加log的详细错误显示
 
-
 ## 2021.07.29 更新
 1. 修复异常抛出时泄露源码路径的bug
 2. 修复原sql的groups字段bug
@@ -154,7 +183,7 @@ mailgun_key = ""
 5. 修复注册用户时没有创建md5密码的bug
 
 ## 2021.04.13 更新
-1. 添加代理功能，目前暂时不可用(By billypon)
+1. 添加proxy功能，目前暂时不可用(By billypon)
 
 ## 2021.02.20 更新
 1. 完善MD5

backup.py

@@ -35,9 +35,9 @@ def new(self, userid, maindb):
                 `ctime` INT UNSIGNED NOT NULL,
                 `mtime` INT UNSIGNED NOT NULL,
                 `atime` INT UNSIGNED NOT NULL,
-                `cip` INT UNSIGNED NOT NULL,
-                `mip` INT UNSIGNED NOT NULL,
-                `aip` INT UNSIGNED NOT NULL,
+                `cip` VARBINARY(16) NOT NULL,
+                `mip` VARBINARY(16) NOT NULL,
+                `aip` VARBINARY(16) NOT NULL,
                 `skey` VARBINARY(128) NOT NULL DEFAULT '',
                 `barkurl` VARBINARY(128) NOT NULL DEFAULT '',
                 `wxpusher` VARBINARY(128) NOT NULL DEFAULT '',

config.py

@@ -43,7 +43,12 @@ class redis(object):
 aes_key = hashlib.sha256(os.getenv('AES_KEY', 'binux').encode('utf-8')).digest()
 cookie_secret = hashlib.sha256(os.getenv('COOKIE_SECRET', 'binux').encode('utf-8')).digest()
 check_task_loop = 500
+# Tornado httpclient.HTTPRequest参数配置
 download_size_limit = 5*1024*1024
+request_timeout = 30.0
+connect_timeout = 30.0
+# delay 延时API最大时间限制，请小于上述timeout配置，否则会报599错误
+delay_max_timeout = 29.9
 proxies = []
 
 # 域名

db/user.py

@@ -46,8 +46,9 @@ def add(self, email, password, ip):
             raise self.DeplicateUser('duplicate username')
 
         now = time.time()
-        if isinstance(ip, str):
-            ip = utils.ip2int(ip)
+        if isinstance(ip,str):
+            ipVersion = utils.isIP(ip)
+            ip = utils.ip2varbinary(ip,ipVersion)
         userkey = umsgpack.unpackb(crypto.password_hash(password))[0]
 
         hash = MD5.new()

libs/fetcher.py

@@ -60,6 +60,8 @@ def _render(obj, key):
         _render(request, 'url')
         for header in request['headers']:
             _render(header, 'name')
+            if pycurl and header['name'][0] == ":":
+                header['name'] = header['name'][1:]
             _render(header, 'value')
             header['value'] = utils.quote_chinese(header['value'])
         for cookie in request['cookies']:
@@ -69,7 +71,7 @@ def _render(obj, key):
         _render(request, 'data')
         return request
 
-    def build_request(self, obj, download_size_limit=config.download_size_limit):
+    def build_request(self, obj, download_size_limit=config.download_size_limit, connect_timeout=config.connect_timeout, request_timeout=config.request_timeout):
         env = obj['env']
         rule = obj['rule']
         request = self.render(obj['request'], env['variables'], env['session'])
@@ -93,6 +95,8 @@ def size_limit(download_size, downloaded, upload_size, uploaded):
                 return 0
             curl.setopt(pycurl.NOPROGRESS, 0)
             curl.setopt(pycurl.PROGRESSFUNCTION, size_limit)
+            curl.setopt(pycurl.CONNECTTIMEOUT, int(connect_timeout))
+            curl.setopt(pycurl.TIMEOUT, int(request_timeout))
             return curl
 
         req = httpclient.HTTPRequest(
@@ -107,6 +111,8 @@ def size_limit(download_size, downloaded, upload_size, uploaded):
                 allow_ipv6 = True,
                 prepare_curl_callback = set_size_limit_callback,
                 validate_cert=False,
+                connect_timeout=connect_timeout,
+                request_timeout=request_timeout
                 )
 
         session = cookie_utils.CookieSession()
@@ -229,7 +235,7 @@ def getdata(_from):
                         content[0] = utils.decode(response.body)
                 if ('content-type' in response.headers):
                     if 'image' in response.headers.get('content-type'):
-                        return base64.b64encode(response.body)
+                        return base64.b64encode(response.body).decode('utf8') 
                 return content[0]
             elif _from == 'status':
                 return '%s' % response.code