Merge pull request #13 from tarosky/issue/12

Support glob patterns for bypassing minifier
Close #12.

.github/workflows/build.yml

@@ -23,13 +23,13 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: '~${{env.PYTHON_VERSION}}'
     - run: script/recreate-venv
     - name: Generate build number
-      uses: einaregilsson/build-number@v3
+      uses: onyxmueller/build-tag-number@v1
       with:
         token: ${{secrets.GITHUB_TOKEN}}
         prefix: build_number_generator
@@ -42,22 +42,27 @@ jobs:
     - name: Embed build number into code
       run: echo "build-$BUILD_NUMBER" > src/origin/response/VERSION
     - run: script/create-lambda
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       with:
         name: artifact
         path: |
           work/origin-request.zip
           work/origin-response.zip
-    - uses: actions/upload-artifact@v2
+        if-no-files-found: error
+    - uses: actions/upload-artifact@v3
       with:
         name: build-number
         path: work/BUILD_NUMBER
+        if-no-files-found: error
 
   test:
     runs-on: ubuntu-20.04
+    permissions:
+      checks: write
+      pull-requests: write
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: '~${{env.PYTHON_VERSION}}'
     - run: echo "$TEST_ACCESS_KEY_ID" > config/test/access-key-id
@@ -70,10 +75,9 @@ jobs:
     - run: pip install -r requirements.txt
     - run: python -m xmlrunner --output-file work/report.xml src/origin/request/test_index.py 2>&1
     - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action@v1
+      uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()
       with:
-        github_token: ${{secrets.GITHUB_TOKEN}}
         files: work/report.xml
 
   release:
@@ -83,10 +87,10 @@ jobs:
     if: github.ref == 'refs/heads/master'
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v3
       with:
         name: artifact
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v3
       with:
         name: build-number
     - name: set BUILD_NUMBER
@@ -116,7 +120,7 @@ jobs:
         asset_path: ./origin-response.zip
         asset_name: origin-response.build-${{env.BUILD_NUMBER}}-awslambda-python3.9.zip
         asset_content_type: application/octet-stream
-    - uses: aws-actions/configure-aws-credentials@v1
+    - uses: aws-actions/configure-aws-credentials@v2
       with:
         aws-access-key-id: ${{env.RELEASE_ACCESS_KEY_ID}}
         aws-secret-access-key: ${{secrets.RELEASE_AWS_SECRET_ACCESS_KEY}}

.python-version

@@ -1,2 +1 @@
-3.9.8
-2.7.18
+3.9.16

requirements.txt

@@ -1,16 +1,18 @@
 autopep8==1.5.7
-boto3==1.20.8
+boto3-stubs[s3,sqs]==1.26.123
+boto3==1.26.123
 flake8==3.9.2
 isort==5.9.3
-mypy_boto3_s3==1.20.1
-mypy_boto3_sqs==1.20.1
+mypy_boto3_s3==1.26.116
+mypy_boto3_sqs==1.26.0.post1
 mypy-boto3-builder==5.5.0
 mypy==0.910
+pathspec==0.11.1
 pep8==1.7.1
 python-dateutil==2.8.2
-python-json-logger==2.0.2
-pytz==2021.3
-types-python-dateutil==2.8.2
-types-pytz==2021.3.0
+python-json-logger==2.0.7
+pytz==2023.3
+types-python-dateutil==2.8.19.12
+types-pytz==2023.3.0.0
 unittest-xml-reporting==3.0.2
 yapf==0.31.0

src/origin/request/index.py

@@ -15,6 +15,8 @@
 from dateutil import parser, tz
 from mypy_boto3_s3.client import S3Client
 from mypy_boto3_sqs.client import SQSClient
+from pathspec import PathSpec
+from pathspec.patterns.gitwildmatch import GitWildMatchPattern
 from pythonjsonlogger.jsonlogger import JsonFormatter
 
 TIMESTAMP_METADATA = 'original-timestamp'
@@ -123,6 +125,7 @@ class XParams:
   sqs_queue_url: str
   perm_resp_max_age: int
   temp_resp_max_age: int
+  bypass_minifier_patterns: str
   expiration_margin: int
 
 
@@ -157,6 +160,10 @@ def json_dump(obj: Any) -> str:
 
 
 def is_not_found_client_error(exception: ClientError) -> bool:
+  if 'Error' not in exception.response:
+    return False
+  if 'Code' not in exception.response['Error']:
+    return False
   return exception.response['Error']['Code'] == '404'
 
 
@@ -188,6 +195,7 @@ def __init__(
       sqs_queue_url: str,
       perm_resp_max_age: int,
       temp_resp_max_age: int,
+      bypass_minifier_path_spec: Optional[PathSpec],
       expiration_margin: int,
   ):
     self.log = log
@@ -201,6 +209,7 @@ def __init__(
     self.sqs_queue_url = sqs_queue_url
     self.perm_resp_max_age = perm_resp_max_age
     self.temp_resp_max_age = temp_resp_max_age
+    self.bypass_minifier_path_spec = bypass_minifier_path_spec
     self.expiration_margin = datetime.timedelta(seconds=expiration_margin)
 
   @classmethod
@@ -219,6 +228,10 @@ def from_lambda(
       sqs_queue_url: str = get_header(req, 'x-env-sqs-queue-url')
       perm_resp_max_age: int = int(get_header(req, 'x-env-perm-resp-max-age'))
       temp_resp_max_age: int = int(get_header(req, 'x-env-temp-resp-max-age'))
+      bypass_minifier_patterns: str = (
+          get_header(req, 'x-bypass-minifier-patterns')
+          if 'x-bypass-minifier-patterns'
+          in req['origin']['s3']['customHeaders'] else '')
     except KeyError as e:
       log.warning({
           MESSAGE: 'environment variable not found',
@@ -234,11 +247,15 @@ def from_lambda(
         sqs_queue_url=sqs_queue_url,
         perm_resp_max_age=perm_resp_max_age,
         temp_resp_max_age=temp_resp_max_age,
+        bypass_minifier_patterns=bypass_minifier_patterns,
         expiration_margin=expiration_margin)
 
     if server_key not in cls.instances:
       sqs = boto3.client('sqs', region_name=region)
       s3 = boto3.client('s3', region_name=region)
+      path_spec = (
+          None if bypass_minifier_patterns == '' else PathSpec.from_lines(
+              GitWildMatchPattern, bypass_minifier_patterns.split(',')))
       cls.instances[server_key] = cls(
           log=log,
           region=region,
@@ -250,6 +267,7 @@ def from_lambda(
           sqs_queue_url=sqs_queue_url,
           perm_resp_max_age=perm_resp_max_age,
           temp_resp_max_age=temp_resp_max_age,
+          bypass_minifier_path_spec=path_spec,
           expiration_margin=expiration_margin)
 
     return cls.instances[server_key]
@@ -424,6 +442,10 @@ def respond_with_original(self) -> FieldUpdate:
     return self.as_overridable(self.as_permanent(FieldUpdate()))
 
   def process(self, path: str, accept_header: str) -> FieldUpdate:
+    if self.bypass_minifier_path_spec is not None and (
+        self.bypass_minifier_path_spec.match_file(path)):
+      return self.respond_with_original()
+
     ext = get_normalized_extension(path)
 
     if ext in image_exts:

src/origin/request/requirements.txt

@@ -1,8 +1,10 @@
-boto3==1.20.8
-mypy_boto3_s3==1.20.1
-mypy_boto3_sqs==1.20.1
+boto3==1.26.123
+boto3-stubs[s3,sqs]==1.26.123
+mypy_boto3_s3==1.26.116
+mypy_boto3_sqs==1.26.0.post1
+pathspec==0.11.1
 python-dateutil==2.8.2
-python-json-logger==2.0.2
-pytz==2021.3
-types-python-dateutil==2.8.2
-types-pytz==2021.3.0
+python-json-logger==2.0.7
+pytz==2023.3
+types-python-dateutil==2.8.19.12
+types-pytz==2023.3.0.0

src/origin/request/test_index.py

@@ -13,6 +13,8 @@
 import boto3
 import pytz
 from mypy_boto3_sqs.type_defs import MessageTypeDef
+from pathspec import PathSpec
+from pathspec.patterns.gitwildmatch import GitWildMatchPattern
 
 from .index import TIMESTAMP_METADATA, FieldUpdate, ImgServer, MyJsonFormatter
 
@@ -22,7 +24,7 @@
 REGION = 'us-east-1'
 
 CSS_MIME = 'text/css'
-GIF_MIME = "image/gif"
+GIF_MIME = 'image/gif'
 JPEG_MIME = 'image/jpeg'
 JS_MIME = 'text/javascript'
 PNG_MIME = 'image/png'
@@ -41,6 +43,8 @@
 JPG_WEBP_NAME_MB_Q = '%E3%83%86%E3%82%B9%E3%83%88.jpg.webp'
 JS_NAME = 'フィズバズ.js'
 JS_NAME_Q = '%E3%83%95%E3%82%A3%E3%82%BA%E3%83%90%E3%82%BA.js'
+JS_NOMINIFY_NAME = 'nominify/foo/bar/image.js'
+JS_NOMINIFY2_NAME = 'no-minify.js'
 MIN_CSS_NAME = 'スタイル.min.css'
 MIN_CSS_NAME_Q = '%E3%82%B9%E3%82%BF%E3%82%A4%E3%83%AB.min.css'
 MIN_JS_NAME = 'スクリプト.min.js'
@@ -58,6 +62,13 @@
 CACHE_CONTROL_TEMP = f'public, max-age={TEMP_RESP_MAX_AGE}'
 
 
+def get_bypass_minifier_patterns(key_prefix: str) -> list[str]:
+  return [
+      f'/{key_prefix}nominify/**',
+      '**/no-minify.js',
+  ]
+
+
 def read_test_config(name: str) -> str:
   path = f'{os.getcwd()}/config/test/{name}'
   with open(path, 'r') as f:
@@ -69,7 +80,8 @@ def generate_safe_random_string() -> str:
 
 
 def create_img_server(
-    log: Logger, name: str, expiration_margin: int) -> ImgServer:
+    log: Logger, name: str, expiration_margin: int,
+    key_prefix: str) -> ImgServer:
   account_id = read_test_config('aws-account-id')
   sqs_name = f'test-{name}-{generate_safe_random_string()}'
 
@@ -94,6 +106,8 @@ def create_img_server(
           f'https://sqs.{REGION}.amazonaws.com/{account_id}/{sqs_name}'),
       perm_resp_max_age=PERM_RESP_MAX_AGE,
       temp_resp_max_age=TEMP_RESP_MAX_AGE,
+      bypass_minifier_path_spec=PathSpec.from_lines(
+          GitWildMatchPattern, get_bypass_minifier_patterns(key_prefix)),
       expiration_margin=expiration_margin)
 
 
@@ -105,8 +119,9 @@ def create_test_environment(
     log: Logger,
     name: str,
     expiration_margin: int,
+    key_prefix: str,
 ) -> ImgServer:
-  img_server = create_img_server(log, name, expiration_margin)
+  img_server = create_img_server(log, name, expiration_margin, key_prefix)
   img_server.sqs.create_queue(
       QueueName=get_test_sqs_queue_name_from_url(img_server.sqs_queue_url))
   return img_server
@@ -171,10 +186,10 @@ def receive_sqs_message(img_server: ImgServer) -> Optional[Dict[str, Any]]:
       VisibilityTimeout=1,
       WaitTimeSeconds=1)
   msgs: Optional[List[MessageTypeDef]] = res.get('Messages', None)
-  if msgs is None or len(msgs) == 0:
+  if msgs is None or len(msgs) == 0 or 'Body' not in msgs[0]:
     return None
-  body: str = msgs[0]['Body']
-  obj: Dict[str, Any] = json.loads(body)
+
+  obj: Dict[str, Any] = json.loads(msgs[0]['Body'])
   return obj
 
 
@@ -196,7 +211,7 @@ def setUp(self) -> None:
     self._log.addHandler(log_handler)
 
     self._img_server = create_test_environment(
-        self._log, 'imglambda', self.get_expiration_margin())
+        self._log, 'imglambda', self.get_expiration_margin(), self._key_prefix)
 
   def get_expiration_margin(self) -> int:
     return 10
@@ -292,6 +307,25 @@ def jpg_accepted_gen_orig(
   #
   # Test_PublicContentJPG
 
+  # Test_BypassMinifierJS
+  def test_bypass_minifier_js_1(self) -> None:
+    update = self._img_server.process(
+        self.to_path(JS_NOMINIFY_NAME), CHROME_ACCEPT_HEADER)
+    self.assertEqual(
+        FieldUpdate(
+            res_cache_control=CACHE_CONTROL_PERM,
+            res_cache_control_overridable='true'), update)
+    self.assert_no_sqs_message()
+
+  def test_bypass_minifier_js_2(self) -> None:
+    update = self._img_server.process(
+        self.to_path(JS_NOMINIFY2_NAME), CHROME_ACCEPT_HEADER)
+    self.assertEqual(
+        FieldUpdate(
+            res_cache_control=CACHE_CONTROL_PERM,
+            res_cache_control_overridable='true'), update)
+    self.assert_no_sqs_message()
+
   # Test_JPGAcceptedS3NoEFS_L
   def test_jpg_accepted_gen_no_orig_l(self) -> None:
     self.jpg_accepted_gen_no_orig(JPG_WEBP_NAME, JPG_NAME, JPG_NAME)

src/origin/response/requirements.txt

@@ -1 +1 @@
-python-json-logger==2.0.2
+python-json-logger==2.0.7