Temp fix CI: add py39, skip py27, disable email & telegram alert (#184)

ENABLE_EMAIL_ALERT and ENABLE_TELEGRAM_ALERT set to False in the Environment Variables of CircleCI.

.circleci/config.yml

@@ -177,18 +177,11 @@ jobs:
     <<: *test-template
     docker:
       - image: circleci/python:2.7
-  py27-sqlite:
-    <<: *test-template
-    docker:
-      - image: circleci/python:2.7
-    environment:
-      SCRAPYDWEB_TESTMODE: True
-      DATA_PATH: '/home/circleci/repo/scrapydweb_data'
-      DATABASE_URL: 'sqlite:////home/circleci/repo/scrapydweb_database'
-  py27-postgresql:
+
+  py37-git-postgresql:
     <<: *test-template
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.7
       # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/
       - image: circleci/postgres:9.6-alpine
         environment:
@@ -199,11 +192,12 @@ jobs:
           POSTGRES_PASSWORD: passw0rd
     environment:
       SCRAPYDWEB_TESTMODE: True
-      DATABASE_URL: 'postgres://circleci:fakepassword@localhost:5432'
-  py27-mysql:
+      DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432'
+
+  py37-git-mysql:
     <<: *test-template
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.7
       # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project
       # Plugin caching_sha2_password could not be loaded
       # - image: circleci/mysql:8.0.4
@@ -218,10 +212,19 @@ jobs:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'mysql://root:rootpw@127.0.0.1:3306'
 
-  py37-git-postgresql:
+  py38-sqlite:
     <<: *test-template
     docker:
-      - image: circleci/python:3.7
+      - image: circleci/python:3.8
+    environment:
+      SCRAPYDWEB_TESTMODE: True
+      DATA_PATH: '/home/circleci/repo/scrapydweb_data'
+      DATABASE_URL: 'sqlite:////home/circleci/repo/scrapydweb_database'
+
+  py38-postgresql:
+    <<: *test-template
+    docker:
+      - image: circleci/python:3.8
       # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/
       - image: circleci/postgres:9.6-alpine
         environment:
@@ -232,11 +235,12 @@ jobs:
           POSTGRES_PASSWORD: passw0rd
     environment:
       SCRAPYDWEB_TESTMODE: True
-      DATABASE_URL: 'postgres://circleci:fakepassword@localhost:5432'
-  py37-git-mysql:
+      DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432'
+
+  py38-mysql:
     <<: *test-template
     docker:
-      - image: circleci/python:3.7
+      - image: circleci/python:3.8
       # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project
       # Plugin caching_sha2_password could not be loaded
       # - image: circleci/mysql:8.0.4
@@ -251,31 +255,17 @@ jobs:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'mysql://root:rootpw@127.0.0.1:3306'
 
-  py38-sqlite:
+  py39:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
-    environment:
-      SCRAPYDWEB_TESTMODE: True
-      DATA_PATH: '/home/circleci/repo/scrapydweb_data'
-      DATABASE_URL: 'sqlite:////home/circleci/repo/scrapydweb_database'
+      - image: circleci/python:3.9
 
 
 workflows:
   test:
     jobs:
-      - py27:
-          is-py27: true
-      - py27-sqlite:
-          is-py27: true
-          set-data-path: true
-          use-sqlite: true
-      - py27-postgresql:
-          is-py27: true
-          use-postgresql: true
-      - py27-mysql:
-          is-py27: true
-          use-mysql: true
+      # - py27:
+      #    is-py27: true
 
       - py37
       - py37-git-postgresql:
@@ -288,3 +278,10 @@ workflows:
       - py38-sqlite:
           set-data-path: true
           use-sqlite: true
+      - py38-postgresql:
+          use-postgresql: true
+      - py38-mysql:
+          use-mysql: true
+
+      - py39
+

scrapydweb/default_settings.py

@@ -362,7 +362,7 @@
 # To use PostgreSQL backend, run command: pip install --upgrade psycopg2
 # e.g.
 # 'mysql://username:password@127.0.0.1:3306'
-# 'postgres://username:password@127.0.0.1:5432'
+# 'postgresql://username:password@127.0.0.1:5432'
 # 'sqlite:///C:/Users/username'
 # 'sqlite:////home/username'
 DATABASE_URL = os.environ.get('DATABASE_URL', '')

scrapydweb/utils/setup_database.py

@@ -11,7 +11,7 @@
 DBS = [DB_APSCHEDULER, DB_TIMERTASKS, DB_METADATA, DB_JOBS]
 
 PATTERN_MYSQL = re.compile(r'mysql://(.+?)(?::(.+?))?@(.+?):(\d+)')
-PATTERN_POSTGRESQL = re.compile(r'postgres://(.+?)(?::(.+?))?@(.+?):(\d+)')
+PATTERN_POSTGRESQL = re.compile(r'(?:postgres|postgresql)://(.+?)(?::(.+?))?@(.+?):(\d+)')
 PATTERN_SQLITE = re.compile(r'sqlite:///(.+)$')
 
 SCRAPYDWEB_TESTMODE = os.environ.get('SCRAPYDWEB_TESTMODE', 'False').lower() == 'true'

tests/test_schedule.py

@@ -139,7 +139,7 @@ def test_telnet_in_stats(app, client):
     desktop_ins = [">Log analysis</li>", ">Log categorization</li>", ">View log</li>", ">Progress visualization</li>"]
     mobile_ins = [">Analysis</li>", ">Categories</li>", ">Charts</li>", ">Logs</li>"]
     telnet_ins = [">Crawler.stats</li>", "<td>datetime.datetime(",
-                  ">Crawler.engine</li>", "<th>engine.has_capacity()</th>", "<td>telnet</td>"]
+                  ">Crawler.engine</li>", "<th>engine.scraper.is_idle()</th>", "<td>telnet</td>"]
     telnet_nos = ["CRITICAL: Unhandled Error", "telnet.OptionRefused"]
     req(app, client, view='schedule.run', kws=dict(node=NODE), data=run_data, ins="run results - ScrapydWeb")
 

tests/test_send_text.py

@@ -9,8 +9,8 @@ def test_send_text_page(app, client):
 
 
 def test_email_pass(app, client):
-    if not app.config.get('EMAIL_PASSWORD', ''):
-        print("EMAIL_PASSWORD empty")
+    if not (app.config.get('ENABLE_EMAIL_ALERT', True) and app.config.get('EMAIL_PASSWORD', '')):
+        print("ENABLE_EMAIL_ALERT False or EMAIL_PASSWORD empty, skip test.")
         return
 
     def check_pass(recipients=None, subject='Email from #scrapydweb', text=None):