diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index c705019b..52195876 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -60,14 +60,107 @@ jobs:
QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
QINIU_TEST_ENV: "travis"
MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
- PYTHONPATH: "$PYTHONPATH:."
run: |
flake8 --show-source --max-line-length=160 ./qiniu
- coverage run -m pytest ./test_qiniu.py ./tests/cases
- ocular --data-file .coverage
- codecov
+ python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+ - name: Post Setup mock server
+ if: ${{ always() }}
+ shell: bash
+ run: |
+ set +e
cat mock-server.pid | xargs kill
+ rm mock-server.pid
- name: Print mock server log
if: ${{ failure() }}
run: |
cat py-mock-server.log
+ - name: Upload results to Codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ test-win:
+ strategy:
+ fail-fast: false
+ max-parallel: 1
+ matrix:
+ python_version: ['2.7', '3.5', '3.9']
+ runs-on: windows-2019
+ # make sure only one test running,
+ # remove this when cases could run in parallel.
+ needs: test
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.ref }}
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ channels: conda-forge
+ python-version: ${{ matrix.python_version }}
+ activate-environment: qiniu-sdk
+ auto-activate-base: false
+ - name: Setup pip
+ env:
+ PYTHON_VERSION: ${{ matrix.python_version }}
+ PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip
+ run: |
+ # reinstall pip by some python(<3.7) not compatible
+ $pyversion = [Version]"$ENV:PYTHON_VERSION"
+ if ($pyversion -lt [Version]"3.7") {
+ Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py"
+ python $ENV:TEMP\get-pip.py --user
+ Remove-Item -Path "$ENV:TEMP\get-pip.py"
+ }
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -I -e ".[dev]"
+ - name: Run cases
+ env:
+ QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
+ QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
+ QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+ QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
+ QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
+ QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
+ QINIU_TEST_ENV: "github"
+ MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
+ PYTHONPATH: "$PYTHONPATH:."
+ run: |
+ Write-Host "======== Setup Mock Server ========="
+ conda create -y -n mock-server python=3.10
+ conda activate mock-server
+ python --version
+ $processOptions = @{
+ FilePath="python"
+ ArgumentList="tests\mock_server\main.py", "--port", "9000"
+ PassThru=$true
+ RedirectStandardOutput="py-mock-server.log"
+ }
+ $mocksrvp = Start-Process @processOptions
+ $mocksrvp.Id | Out-File -FilePath "mock-server.pid"
+ conda deactivate
+ Sleep 3
+ Write-Host "======== Running Test ========="
+ python --version
+ python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+ - name: Post Setup mock server
+ if: ${{ always() }}
+ run: |
+ Try {
+ $mocksrvpid = Get-Content -Path "mock-server.pid"
+ Stop-Process -Id $mocksrvpid
+ Remove-Item -Path "mock-server.pid"
+ } Catch {
+ Write-Host -Object $_
+ }
+ - name: Print mock server log
+ if: ${{ failure() }}
+ run: |
+ Get-Content -Path "py-mock-server.log" | Write-Host
+ - name: Upload results to Codecov
+ uses: codecov/codecov-action@v4
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/codecov.yml b/codecov.yml
index 3f36c50a..0aab28d3 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1,14 +1,14 @@
codecov:
ci:
- - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要
- require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。
+ - prow.qiniu.io # prow need this. seems useless
+ require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed
-github_checks: #关闭github checks
+github_checks: # close github checks
annotations: false
comment:
layout: "reach, diff, flags, files"
- behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。
+ behavior: new # `new` means the bot will comment a new message instead of edit the old one
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
@@ -16,13 +16,13 @@ comment:
- "master"
coverage:
- status: # 评判 pr 通过的标准
+ status: # check coverage status to pass or fail
patch: off
- project: # project 统计所有代码x
+ project: # project analyze all code in the project
default:
# basic
- target: 73.5% # 总体通过标准
- threshold: 3% # 允许单次下降的幅度
+ target: 73.5% # the minimum coverage ratio that the commit must meet
+ threshold: 3% # allow the coverage to drop
base: auto
if_not_found: success
if_ci_failed: error
diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py
index 8b52822c..13d1800a 100644
--- a/qiniu/http/regions_provider.py
+++ b/qiniu/http/regions_provider.py
@@ -5,9 +5,10 @@
import logging
import tempfile
import os
+import shutil
from qiniu.compat import json, b as to_bytes
-from qiniu.utils import io_md5
+from qiniu.utils import io_md5, dt2ts
from .endpoint import Endpoint
from .region import Region, ServiceName
@@ -264,7 +265,7 @@ def _persist_region(region):
},
ttl=region.ttl,
# use datetime.datetime.timestamp() when min version of python >= 3
- createTime=int(float(region.create_time.strftime('%s.%f')) * 1000)
+ createTime=dt2ts(region.create_time)
)._asdict()
@@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
with open(persist_path, 'r') as f:
for line in f:
+ if not line.strip():
+ continue
try:
- cache_key, regions = _parse_persisted_regions(line)
+ cache_key, regions = _parse_persisted_regions(line.strip())
yield cache_key, regions
except Exception as err:
if not ignore_parse_error:
@@ -655,7 +658,7 @@ def __shrink_cache(self):
)
# rename file
- os.rename(shrink_file_path, self._cache_scope.persist_path)
+ shutil.move(shrink_file_path, self._cache_scope.persist_path)
except FileAlreadyLocked:
pass
finally:
diff --git a/qiniu/region.py b/qiniu/region.py
index 09ac791d..a59d488e 100644
--- a/qiniu/region.py
+++ b/qiniu/region.py
@@ -6,7 +6,7 @@
from .compat import json, s as str_from_bytes
-from .utils import urlsafe_base64_decode
+from .utils import urlsafe_base64_decode, dt2ts
from .config import UC_HOST, is_customized_default, get_default
from .http.endpoint import Endpoint as _HTTPEndpoint
from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider
@@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False):
ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day
# use datetime.datetime.timestamp() when min version of python >= 3
- create_time = int(float(region.create_time.strftime('%s.%f')) * 1000)
+ create_time = dt2ts(region.create_time)
bucket_hosts['deadline'] = create_time + ttl
return bucket_hosts
diff --git a/qiniu/utils.py b/qiniu/utils.py
index f8517e35..197b8813 100644
--- a/qiniu/utils.py
+++ b/qiniu/utils.py
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
from hashlib import sha1, new as hashlib_new
from base64 import urlsafe_b64encode, urlsafe_b64decode
-from datetime import datetime
+from datetime import datetime, tzinfo, timedelta
+
from .compat import b, s
try:
@@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name):
result += ch
upper = ch == "-"
return result
+
+
+class _UTC_TZINFO(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=0)
+
+ def tzname(self, dt):
+ return "UTC"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+
+def dt2ts(dt):
+ """
+ converte datetime to timestamp
+
+ Parameters
+ ----------
+ dt: datetime.datetime
+ """
+ if not dt.tzinfo:
+ st = (dt - datetime(1970, 1, 1)).total_seconds()
+ else:
+ st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds()
+
+ return int(st)
diff --git a/setup.py b/setup.py
index cf97eae2..fa920d45 100644
--- a/setup.py
+++ b/setup.py
@@ -42,10 +42,8 @@ def find_version(*file_paths):
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
@@ -66,8 +64,6 @@ def find_version(*file_paths):
'pytest',
'pytest-cov',
'freezegun',
- 'scrutinizer-ocular',
- 'codecov'
]
},
diff --git a/test_qiniu.py b/test_qiniu.py
index 2b71aa22..c8dce456 100644
--- a/test_qiniu.py
+++ b/test_qiniu.py
@@ -65,129 +65,6 @@ def remove_temp_file(file):
except OSError:
pass
-
-class UtilsTest(unittest.TestCase):
- def test_urlsafe(self):
- a = 'hello\x96'
- u = urlsafe_base64_encode(a)
- assert b(a) == urlsafe_base64_decode(u)
-
- def test_canonical_mime_header_key(self):
- field_names = [
- ":status",
- ":x-test-1",
- ":x-Test-2",
- "content-type",
- "CONTENT-LENGTH",
- "oRiGin",
- "ReFer",
- "Last-Modified",
- "acCePt-ChArsEt",
- "x-test-3",
- "cache-control",
- ]
- expect_canonical_field_names = [
- ":status",
- ":x-test-1",
- ":x-Test-2",
- "Content-Type",
- "Content-Length",
- "Origin",
- "Refer",
- "Last-Modified",
- "Accept-Charset",
- "X-Test-3",
- "Cache-Control",
- ]
- assert len(field_names) == len(expect_canonical_field_names)
- for i in range(len(field_names)):
- assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i]
-
- def test_entry(self):
- case_list = [
- {
- 'msg': 'normal',
- 'bucket': 'qiniuphotos',
- 'key': 'gogopher.jpg',
- 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
- },
- {
- 'msg': 'key empty',
- 'bucket': 'qiniuphotos',
- 'key': '',
- 'expect': 'cWluaXVwaG90b3M6'
- },
- {
- 'msg': 'key undefined',
- 'bucket': 'qiniuphotos',
- 'key': None,
- 'expect': 'cWluaXVwaG90b3M='
- },
- {
- 'msg': 'key need replace plus symbol',
- 'bucket': 'qiniuphotos',
- 'key': '012ts>a',
- 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
- },
- {
- 'msg': 'key need replace slash symbol',
- 'bucket': 'qiniuphotos',
- 'key': '012ts?a',
- 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
- }
- ]
- for c in case_list:
- assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg')
-
- def test_decode_entry(self):
- case_list = [
- {
- 'msg': 'normal',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': 'gogopher.jpg'
- },
- 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
- },
- {
- 'msg': 'key empty',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': ''
- },
- 'entry': 'cWluaXVwaG90b3M6'
- },
- {
- 'msg': 'key undefined',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': None
- },
- 'entry': 'cWluaXVwaG90b3M='
- },
- {
- 'msg': 'key need replace plus symbol',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': '012ts>a'
- },
- 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
- },
- {
- 'msg': 'key need replace slash symbol',
- 'expect': {
- 'bucket': 'qiniuphotos',
- 'key': '012ts?a'
- },
- 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
- }
- ]
- for c in case_list:
- bucket, key = decode_entry(c.get('entry'))
- assert bucket == c.get('expect').get('bucket'), c.get('msg')
- assert key == c.get('expect').get('key'), c.get('msg')
-
-
class BucketTestCase(unittest.TestCase):
q = Auth(access_key, secret_key)
bucket = BucketManager(q)
@@ -408,7 +285,11 @@ def test_invalid_x_qiniu_date_with_disable_date_sign(self):
def test_invalid_x_qiniu_date_env(self):
os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True'
ret, info = self.bucket.stat(bucket_name, 'python-sdk.html')
- os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ if hasattr(os, 'unsetenv'):
+ os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ else:
+ # fix unsetenv not exists in earlier python on windows
+ os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = ''
assert 'hash' in ret
@freeze_time("1970-01-01")
@@ -417,7 +298,11 @@ def test_invalid_x_qiniu_date_env_be_ignored(self):
q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False)
bucket = BucketManager(q)
ret, info = bucket.stat(bucket_name, 'python-sdk.html')
- os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ if hasattr(os, 'unsetenv'):
+ os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE')
+ else:
+ # fix unsetenv not exists in earlier python on windows
+ os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = ''
assert ret is None
assert info.status_code == 403
diff --git a/tests/cases/test_http/test_region.py b/tests/cases/test_http/test_region.py
index a66b16c9..976d2619 100644
--- a/tests/cases/test_http/test_region.py
+++ b/tests/cases/test_http/test_region.py
@@ -36,7 +36,7 @@ def test_custom_options(self):
k in region.services
for k in chain(ServiceName, ['custom-service'])
)
- assert datetime.now() - region.create_time > timedelta(days=1)
+ assert datetime.now() - region.create_time >= timedelta(days=1)
assert region.ttl == 3600
assert not region.is_live
diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py
index 163f19d2..7289f5ca 100644
--- a/tests/cases/test_http/test_regions_provider.py
+++ b/tests/cases/test_http/test_regions_provider.py
@@ -186,8 +186,10 @@ def test_getter_with_base_regions_provider(self, cached_regions_provider):
assert list(cached_regions_provider) == regions
line_num = 0
with open(cached_regions_provider.persist_path, 'r') as f:
- for _ in f:
- line_num += 1
+ for l in f:
+ # ignore empty line
+ if l.strip():
+ line_num += 1
assert line_num == 1
@pytest.mark.parametrize(
diff --git a/tests/cases/test_utils.py b/tests/cases/test_utils.py
new file mode 100644
index 00000000..11d9db77
--- /dev/null
+++ b/tests/cases/test_utils.py
@@ -0,0 +1,145 @@
+from datetime import datetime, timedelta, tzinfo
+
+from qiniu import utils, compat
+
+
+class _CN_TZINFO(tzinfo):
+ def utcoffset(self, dt):
+ return timedelta(hours=8)
+
+ def tzname(self, dt):
+ return "CST"
+
+ def dst(self, dt):
+ return timedelta(0)
+
+
+class TestUtils:
+ def test_urlsafe(self):
+ a = 'hello\x96'
+ u = utils.urlsafe_base64_encode(a)
+ assert compat.b(a) == utils.urlsafe_base64_decode(u)
+
+ def test_canonical_mime_header_key(self):
+ field_names = [
+ ":status",
+ ":x-test-1",
+ ":x-Test-2",
+ "content-type",
+ "CONTENT-LENGTH",
+ "oRiGin",
+ "ReFer",
+ "Last-Modified",
+ "acCePt-ChArsEt",
+ "x-test-3",
+ "cache-control",
+ ]
+ expect_canonical_field_names = [
+ ":status",
+ ":x-test-1",
+ ":x-Test-2",
+ "Content-Type",
+ "Content-Length",
+ "Origin",
+ "Refer",
+ "Last-Modified",
+ "Accept-Charset",
+ "X-Test-3",
+ "Cache-Control",
+ ]
+ assert len(field_names) == len(expect_canonical_field_names)
+ for i in range(len(field_names)):
+ assert utils.canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i]
+
+ def test_entry(self):
+ case_list = [
+ {
+ 'msg': 'normal',
+ 'bucket': 'qiniuphotos',
+ 'key': 'gogopher.jpg',
+ 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
+ },
+ {
+ 'msg': 'key empty',
+ 'bucket': 'qiniuphotos',
+ 'key': '',
+ 'expect': 'cWluaXVwaG90b3M6'
+ },
+ {
+ 'msg': 'key undefined',
+ 'bucket': 'qiniuphotos',
+ 'key': None,
+ 'expect': 'cWluaXVwaG90b3M='
+ },
+ {
+ 'msg': 'key need replace plus symbol',
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts>a',
+ 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
+ },
+ {
+ 'msg': 'key need replace slash symbol',
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts?a',
+ 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
+ }
+ ]
+ for c in case_list:
+ assert c.get('expect') == utils.entry(c.get('bucket'), c.get('key')), c.get('msg')
+
+ def test_decode_entry(self):
+ case_list = [
+ {
+ 'msg': 'normal',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': 'gogopher.jpg'
+ },
+ 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
+ },
+ {
+ 'msg': 'key empty',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': ''
+ },
+ 'entry': 'cWluaXVwaG90b3M6'
+ },
+ {
+ 'msg': 'key undefined',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': None
+ },
+ 'entry': 'cWluaXVwaG90b3M='
+ },
+ {
+ 'msg': 'key need replace plus symbol',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts>a'
+ },
+ 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
+ },
+ {
+ 'msg': 'key need replace slash symbol',
+ 'expect': {
+ 'bucket': 'qiniuphotos',
+ 'key': '012ts?a'
+ },
+ 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
+ }
+ ]
+ for c in case_list:
+ bucket, key = utils.decode_entry(c.get('entry'))
+ assert bucket == c.get('expect', {}).get('bucket'), c.get('msg')
+ assert key == c.get('expect', {}).get('key'), c.get('msg')
+
+ def test_dt2ts(self):
+ dt = datetime(year=2011, month=8, day=3, tzinfo=_CN_TZINFO())
+ expect = 1312300800
+ assert utils.dt2ts(dt) == expect
+
+ base_dt = datetime(year=2011, month=8, day=3)
+ now_dt = datetime.now()
+ assert int((now_dt - base_dt).total_seconds()) == utils.dt2ts(now_dt) - utils.dt2ts(base_dt)