Skip to content

Commit d6339c4

Browse files
authored
Get almost the entire results-processor typed (#4060)
1 parent 716c11b commit d6339c4

File tree

8 files changed

+194
-92
lines changed

8 files changed

+194
-92
lines changed

results-processor/gsutil.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,30 +5,34 @@
55
import logging
66
import re
77
import subprocess
8-
8+
from typing import List, Tuple
99

1010
_log = logging.getLogger(__name__)
1111

1212

13-
def _call(command):
13+
def _call(command: List[str]) -> None:
1414
_log.info('EXEC: %s', ' '.join(command))
1515
subprocess.check_call(command)
1616

1717

18-
def split_gcs_path(gcs_path):
18+
def split_gcs_path(gcs_path: str) -> Tuple[str, str]:
1919
"""Splits /bucket/path into (bucket, path)."""
2020
match = re.match(r'/([^/]+)/(.*)', gcs_path)
2121
assert match
22-
return match.groups()
22+
g = match.groups()
23+
assert len(g) == 2
24+
return g
2325

2426

25-
def gs_to_public_url(gs_url):
27+
def gs_to_public_url(gs_url: str) -> str:
2628
"""Converts a gs:// URI to a HTTP URL."""
2729
assert gs_url.startswith('gs://')
2830
return gs_url.replace('gs://', 'https://storage.googleapis.com/', 1)
2931

3032

31-
def copy(path1, path2, gzipped=False, quiet=True):
33+
def copy(
34+
path1: str, path2: str, gzipped: bool = False, quiet: bool = True
35+
) -> None:
3236
"""Copies path1 to path2 with gsutil cp.
3337
3438
Args:

results-processor/main.py

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
import tempfile
66
import time
77
from http import HTTPStatus
8+
from typing import Any, Callable, TypeVar, cast
89

910
import filelock
1011
import flask
12+
from flask.typing import ResponseReturnValue
1113

1214
import processor
1315

14-
1516
# The file will be flock()'ed if a report is being processed.
1617
LOCK_FILE = '/tmp/results-processor.lock'
1718
# If the file above is locked, this timestamp file contains the UNIX timestamp
@@ -32,7 +33,7 @@
3233
app = flask.Flask(__name__)
3334

3435

35-
def _atomic_write(path, content):
36+
def _atomic_write(path: str, content: str) -> None:
3637
# Do not auto-delete the file because we will move it after closing it.
3738
temp = tempfile.NamedTemporaryFile(mode='wt', delete=False)
3839
temp.write(content)
@@ -41,12 +42,15 @@ def _atomic_write(path, content):
4142
os.replace(temp.name, path)
4243

4344

44-
def _serial_task(func):
45+
F = TypeVar('F', bound=Callable[..., Any])
46+
47+
48+
def _serial_task(func: F) -> F:
4549
lock = filelock.FileLock(LOCK_FILE)
4650

4751
# It is important to use wraps() to preserve the original name & docstring.
4852
@functools.wraps(func)
49-
def decorated_func(*args, **kwargs):
53+
def decorated_func(*args: object, **kwargs: object) -> object:
5054
try:
5155
with lock.acquire(timeout=1):
5256
return func(*args, **kwargs)
@@ -55,24 +59,24 @@ def decorated_func(*args, **kwargs):
5559
return ('A result is currently being processed.',
5660
HTTPStatus.SERVICE_UNAVAILABLE)
5761

58-
return decorated_func
62+
return cast(F, decorated_func)
5963

6064

61-
def _internal_only(func):
65+
def _internal_only(func: F) -> F:
6266
@functools.wraps(func)
63-
def decorated_func(*args, **kwargs):
67+
def decorated_func(*args: object, **kwargs: object) -> object:
6468
if (not app.debug and
6569
# This header cannot be set by external requests.
6670
# https://cloud.google.com/tasks/docs/creating-appengine-handlers?hl=en#reading_app_engine_task_request_headers
6771
not flask.request.headers.get('X-AppEngine-QueueName')):
6872
return ('External requests not allowed', HTTPStatus.FORBIDDEN)
6973
return func(*args, **kwargs)
7074

71-
return decorated_func
75+
return cast(F, decorated_func)
7276

7377

7478
@app.route('/_ah/liveness_check')
75-
def liveness_check():
79+
def liveness_check() -> ResponseReturnValue:
7680
lock = filelock.FileLock(LOCK_FILE)
7781
try:
7882
lock.acquire(timeout=0.1)
@@ -91,7 +95,7 @@ def liveness_check():
9195

9296

9397
@app.route('/_ah/readiness_check')
94-
def readiness_check():
98+
def readiness_check() -> ResponseReturnValue:
9599
lock = filelock.FileLock(LOCK_FILE)
96100
try:
97101
lock.acquire(timeout=0.1)
@@ -106,7 +110,7 @@ def readiness_check():
106110
@app.route('/api/results/process', methods=['POST'])
107111
@_internal_only
108112
@_serial_task
109-
def task_handler():
113+
def task_handler() -> ResponseReturnValue:
110114
_atomic_write(TIMESTAMP_FILE, str(time.time()))
111115

112116
task_id = flask.request.headers.get('X-AppEngine-TaskName')

results-processor/mypy.ini

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,22 @@
11
[mypy]
2+
check_untyped_defs = True
3+
disallow_any_generics = True
4+
disallow_incomplete_defs = True
25
disallow_subclassing_any = True
3-
#disallow_any_generics = True
46
disallow_untyped_calls = True
5-
#disallow_untyped_defs = True
6-
disallow_incomplete_defs = True
7-
#check_untyped_defs = True
87
disallow_untyped_decorators = True
9-
no_implicit_optional = True
8+
disallow_untyped_defs = True
9+
extra_checks = True
10+
no_implicit_reexport = True
11+
strict_equality = True
1012
warn_redundant_casts = True
11-
warn_unused_ignores = True
1213
warn_return_any = True
14+
warn_unused_configs = True
15+
warn_unused_ignores = True
1316

14-
[mypy-filelock]
15-
ignore_missing_imports = True
16-
17-
[mypy-google.cloud]
18-
ignore_missing_imports = True
17+
exclude = (?x)(
18+
^.*_test\.py$
19+
| ^test_.*\.py$
20+
)
1921

20-
[mypy-requests]
21-
ignore_missing_imports = True
22+
untyped_calls_exclude = google.cloud.datastore

0 commit comments

Comments
 (0)