From e409f2e4374d02ff1a3213e5e5aa182db15f4383 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 12:13:53 +0300
Subject: [PATCH 01/21] fix: package.py - changed build plan file names
 extension to a .plan.json

---
 package.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.py b/package.py
index ce5b3a8e..9c352e0d 100644
--- a/package.py
+++ b/package.py
@@ -354,7 +354,7 @@ def update_hash(hash_obj, file_root, file_path):
 
     build_plan = json.dumps(build_data)
     build_plan_filename = os.path.join(artifacts_dir,
-                                       '{}.plan'.format(content_hash))
+                                       '{}.plan.json'.format(content_hash))
     if not os.path.exists(artifacts_dir):
         os.makedirs(artifacts_dir)
     with open(build_plan_filename, 'w') as f:

From ee402d0538b1407705c7f992e06b0b11f9f2461a Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 08:42:48 +0300
Subject: [PATCH 02/21] lambda.py: Added hidden 'zip' command for debug purpose

---
 package.py | 60 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 35 insertions(+), 25 deletions(-)

diff --git a/package.py b/package.py
index 9c352e0d..db0b4796 100644
--- a/package.py
+++ b/package.py
@@ -123,7 +123,7 @@ def timestamp_now_ns():
 ################################################################################
 # Packaging functions
 
-def make_zipfile(base_name, base_dir):
+def make_zipfile(zip_filename, base_dir, timestamp=None):
     """
     Create a zip file from all the files under 'base_dir'.
     The output zip file will be named 'base_name' + ".zip".  Returns the
@@ -131,8 +131,7 @@ def make_zipfile(base_name, base_dir):
     """
     logger = logging.getLogger('zip')
 
-    zip_filename = base_name + ".zip"
-    archive_dir = os.path.dirname(base_name)
+    archive_dir = os.path.dirname(zip_filename)
 
     if archive_dir and not os.path.exists(archive_dir):
         logger.info("creating %s", archive_dir)
@@ -505,27 +504,6 @@ def create_zip_file(source_dir, target_file):
         logger.info('Created: %s', shlex.quote(filename))
 
 
-def args_parser():
-    ap = argparse.ArgumentParser()
-    ap.set_defaults(command=lambda _: ap.print_usage())
-    sp = ap.add_subparsers(metavar="COMMAND")
-
-    p = sp.add_parser('prepare',
-                      help='compute a filename hash for a zip archive')
-    p.set_defaults(command=prepare_command)
-
-    p = sp.add_parser('build',
-                      help='build and pack to a zip archive')
-    p.set_defaults(command=build_command)
-    p.add_argument('-t', '--timestamp',
-                   dest='zip_file_timestamp', type=int, required=True,
-                   help='A zip file timestamp generated by the prepare command')
-    p.add_argument('build_plan_file', metavar='PLAN_FILE',
-                   help='A build plan file provided by the prepare command')
-    add_hidden_commands(sp)
-    return ap
-
-
 def add_hidden_commands(sub_parsers):
     sp = sub_parsers
 
@@ -543,7 +521,7 @@ def hidden_parser(name, **kwargs):
     p.add_argument('-r', '--runtime', help='A docker image runtime',
                    default='python3.8')
 
-    p = hidden_parser('docker_image', help='Run docker build')
+    p = hidden_parser('docker-image', help='Run docker build')
     p.set_defaults(command=lambda args: call(docker_build_command(
         args.build_root, args.docker_file, args.tag)))
     p.add_argument('-t', '--tag', help='A docker image tag')
@@ -551,6 +529,38 @@ def hidden_parser(name, **kwargs):
     p.add_argument('docker_file', help='A docker file path',
                    nargs=argparse.OPTIONAL)
 
+    def zip_cmd(args):
+        make_zipfile(args.zipfile, args.dir, args.timestamp)
+        logger.info('-' * 80)
+        subprocess.call(['zipinfo', args.zipfile])
+    p = hidden_parser('zip', help='Zip folder with provided files timestamp')
+    p.set_defaults(command=zip_cmd)
+    p.add_argument('zipfile', help='Path to a zip file')
+    p.add_argument('dir', help='Path to a directory for packaging')
+    p.add_argument('-t', '--timestamp', default=timestamp_now_ns(), type=int,
+                   help='A timestamp to override for all zip members')
+
+
+def args_parser():
+    ap = argparse.ArgumentParser()
+    ap.set_defaults(command=lambda _: ap.print_usage())
+    sp = ap.add_subparsers(metavar="COMMAND")
+
+    p = sp.add_parser('prepare',
+                      help='compute a filename hash for a zip archive')
+    p.set_defaults(command=prepare_command)
+
+    p = sp.add_parser('build',
+                      help='build and pack to a zip archive')
+    p.set_defaults(command=build_command)
+    p.add_argument('-t', '--timestamp',
+                   dest='zip_file_timestamp', type=int, required=True,
+                   help='A zip file timestamp generated by the prepare command')
+    p.add_argument('build_plan_file', metavar='PLAN_FILE',
+                   help='A build plan file provided by the prepare command')
+    add_hidden_commands(sp)
+    return ap
+
 
 def main():
     ns = argparse.Namespace(

From 0bd95749b3097c0ab9b7b74f9f6046ec918baf7b Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 16:31:01 +0300
Subject: [PATCH 03/21] package.py - extracted dir listing logic

---
 package.py | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/package.py b/package.py
index db0b4796..ea8db2a6 100644
--- a/package.py
+++ b/package.py
@@ -123,6 +123,20 @@ def timestamp_now_ns():
 ################################################################################
 # Packaging functions
 
+def emit_dir_files(base_dir):
+    path = os.path.normpath(base_dir)
+    if path != os.curdir:
+        yield path
+    for dirpath, dirnames, filenames in os.walk(base_dir):
+        for name in sorted(dirnames):
+            path = os.path.normpath(os.path.join(dirpath, name))
+            yield path
+        for name in filenames:
+            path = os.path.normpath(os.path.join(dirpath, name))
+            if os.path.isfile(path):
+                yield path
+
+
 def make_zipfile(zip_filename, base_dir, timestamp=None):
     """
     Create a zip file from all the files under 'base_dir'.
@@ -142,20 +156,9 @@ def make_zipfile(zip_filename, base_dir, timestamp=None):
 
     with zipfile.ZipFile(zip_filename, "w",
                          compression=zipfile.ZIP_DEFLATED) as zf:
-        path = os.path.normpath(base_dir)
-        if path != os.curdir:
-            zf.write(path, path)
+        for path in emit_dir_files(base_dir):
             logger.info("adding '%s'", path)
-        for dirpath, dirnames, filenames in os.walk(base_dir):
-            for name in sorted(dirnames):
-                path = os.path.normpath(os.path.join(dirpath, name))
-                zf.write(path, path)
-                logger.info("adding '%s'", path)
-            for name in filenames:
-                path = os.path.normpath(os.path.join(dirpath, name))
-                if os.path.isfile(path):
-                    zf.write(path, path)
-                    logger.info("adding '%s'", path)
+            zf.write(path, path)
     return zip_filename
 
 

From ae4c76fc5e19d6ae1c48116749726de766d61e23 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 16:37:44 +0300
Subject: [PATCH 04/21] package.py - Added zipping of multiple directories

---
 package.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/package.py b/package.py
index ea8db2a6..4597e71b 100644
--- a/package.py
+++ b/package.py
@@ -137,7 +137,7 @@ def emit_dir_files(base_dir):
                 yield path
 
 
-def make_zipfile(zip_filename, base_dir, timestamp=None):
+def make_zipfile(zip_filename, *base_dirs, timestamp=None):
     """
     Create a zip file from all the files under 'base_dir'.
     The output zip file will be named 'base_name' + ".zip".  Returns the
@@ -151,14 +151,15 @@ def make_zipfile(zip_filename, base_dir, timestamp=None):
         logger.info("creating %s", archive_dir)
         os.makedirs(archive_dir)
 
-    logger.info("creating '%s' and adding '%s' to it",
-                zip_filename, base_dir)
+    logger.info("creating '%s' archive", zip_filename)
 
     with zipfile.ZipFile(zip_filename, "w",
                          compression=zipfile.ZIP_DEFLATED) as zf:
-        for path in emit_dir_files(base_dir):
-            logger.info("adding '%s'", path)
-            zf.write(path, path)
+        for base_dir in base_dirs:
+            logger.info("adding directory '%s'", base_dir)
+            for path in emit_dir_files(base_dir):
+                logger.info("adding '%s'", path)
+                zf.write(path, path)
     return zip_filename
 
 
@@ -533,13 +534,14 @@ def hidden_parser(name, **kwargs):
                    nargs=argparse.OPTIONAL)
 
     def zip_cmd(args):
-        make_zipfile(args.zipfile, args.dir, args.timestamp)
+        make_zipfile(args.zipfile, *args.dir, timestamp=args.timestamp)
         logger.info('-' * 80)
         subprocess.call(['zipinfo', args.zipfile])
     p = hidden_parser('zip', help='Zip folder with provided files timestamp')
     p.set_defaults(command=zip_cmd)
     p.add_argument('zipfile', help='Path to a zip file')
-    p.add_argument('dir', help='Path to a directory for packaging')
+    p.add_argument('dir', nargs=argparse.ONE_OR_MORE,
+                   help='Path to a directory for packaging')
     p.add_argument('-t', '--timestamp', default=timestamp_now_ns(), type=int,
                    help='A timestamp to override for all zip members')
 

From bc86c5718745bd71420c31fd216d00e635fc0c65 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 16:41:14 +0300
Subject: [PATCH 05/21] package.py - make compression definable by parameter

---
 package.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.py b/package.py
index 4597e71b..76e084a2 100644
--- a/package.py
+++ b/package.py
@@ -137,7 +137,8 @@ def emit_dir_files(base_dir):
                 yield path
 
 
-def make_zipfile(zip_filename, *base_dirs, timestamp=None):
+def make_zipfile(zip_filename, *base_dirs, timestamp=None,
+                 compression=zipfile.ZIP_DEFLATED):
     """
     Create a zip file from all the files under 'base_dir'.
     The output zip file will be named 'base_name' + ".zip".  Returns the
@@ -153,8 +154,7 @@ def make_zipfile(zip_filename, *base_dirs, timestamp=None):
 
     logger.info("creating '%s' archive", zip_filename)
 
-    with zipfile.ZipFile(zip_filename, "w",
-                         compression=zipfile.ZIP_DEFLATED) as zf:
+    with zipfile.ZipFile(zip_filename, "w", compression) as zf:
         for base_dir in base_dirs:
             logger.info("adding directory '%s'", base_dir)
             for path in emit_dir_files(base_dir):

From 9e250e8b7aeb4c593101699c98ea79262d363a89 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 17:19:06 +0300
Subject: [PATCH 06/21] package.py - Added patched ZipFile.write function

---
 package.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 59 insertions(+), 1 deletion(-)

diff --git a/package.py b/package.py
index 76e084a2..ed06df3c 100644
--- a/package.py
+++ b/package.py
@@ -144,6 +144,64 @@ def make_zipfile(zip_filename, *base_dirs, timestamp=None,
     The output zip file will be named 'base_name' + ".zip".  Returns the
     name of the output zip file.
     """
+    if timestamp:
+        timestamp = datetime.datetime.fromtimestamp(timestamp).timetuple()[:6]
+
+    # An extended version of a write method
+    # from the original zipfile.py library module
+    def write(self, filename, arcname=None,
+              compress_type=None, compresslevel=None,
+              date_time=None):
+        """Put the bytes from filename into the archive under the name
+        arcname."""
+        if not self.fp:
+            raise ValueError(
+                "Attempt to write to ZIP archive that was already closed")
+        if self._writing:
+            raise ValueError(
+                "Can't write to ZIP archive while an open writing handle exists"
+            )
+
+        zinfo = zipfile.ZipInfo.from_file(
+            filename, arcname, strict_timestamps=self._strict_timestamps)
+
+        if date_time:
+            zinfo.date_time = date_time
+
+        if zinfo.is_dir():
+            zinfo.compress_size = 0
+            zinfo.CRC = 0
+        else:
+            if compress_type is not None:
+                zinfo.compress_type = compress_type
+            else:
+                zinfo.compress_type = self.compression
+
+            if compresslevel is not None:
+                zinfo._compresslevel = compresslevel
+            else:
+                zinfo._compresslevel = self.compresslevel
+
+        if zinfo.is_dir():
+            with self._lock:
+                if self._seekable:
+                    self.fp.seek(self.start_dir)
+                zinfo.header_offset = self.fp.tell()  # Start of header bytes
+                if zinfo.compress_type == zipfile.ZIP_LZMA:
+                # Compressed data includes an end-of-stream (EOS) marker
+                    zinfo.flag_bits |= 0x02
+
+                self._writecheck(zinfo)
+                self._didModify = True
+
+                self.filelist.append(zinfo)
+                self.NameToInfo[zinfo.filename] = zinfo
+                self.fp.write(zinfo.FileHeader(False))
+                self.start_dir = self.fp.tell()
+        else:
+            with open(filename, "rb") as src, self.open(zinfo, 'w') as dest:
+                shutil.copyfileobj(src, dest, 1024*8)
+
     logger = logging.getLogger('zip')
 
     archive_dir = os.path.dirname(zip_filename)
@@ -159,7 +217,7 @@ def make_zipfile(zip_filename, *base_dirs, timestamp=None,
             logger.info("adding directory '%s'", base_dir)
             for path in emit_dir_files(base_dir):
                 logger.info("adding '%s'", path)
-                zf.write(path, path)
+                write(zf, path, path)
     return zip_filename
 
 

From d0c9c3c5be534cf67958469918e362474d1c8104 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 17:39:16 +0300
Subject: [PATCH 07/21] package.py - Added possibility to adapt archive entity
 timestamps

Usage:
  > python package.py zip foo.zip examples/simple examples/fixtures -v -t `date +%s`
  > python package.py zip foo.zip examples/fixtures -v -t `date +%s -d '- 4 days'`
---
 package.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/package.py b/package.py
index ed06df3c..74c97f02 100644
--- a/package.py
+++ b/package.py
@@ -144,8 +144,6 @@ def make_zipfile(zip_filename, *base_dirs, timestamp=None,
     The output zip file will be named 'base_name' + ".zip".  Returns the
     name of the output zip file.
     """
-    if timestamp:
-        timestamp = datetime.datetime.fromtimestamp(timestamp).timetuple()[:6]
 
     # An extended version of a write method
     # from the original zipfile.py library module
@@ -202,8 +200,25 @@ def write(self, filename, arcname=None,
             with open(filename, "rb") as src, self.open(zinfo, 'w') as dest:
                 shutil.copyfileobj(src, dest, 1024*8)
 
+    def str_int_to_timestamp(s):
+        return int(s) / 10 ** (len(s) - 10)
+
     logger = logging.getLogger('zip')
 
+    date_time = None
+    if timestamp is not None:
+        if isinstance(timestamp, str):
+            if timestamp.isnumeric():
+                timestamp = str_int_to_timestamp(timestamp)
+            else:
+                timestamp = float(timestamp)
+        elif isinstance(timestamp, int):
+            timestamp = str_int_to_timestamp(str(timestamp))
+
+        date_time = datetime.datetime.fromtimestamp(timestamp).timetuple()[:6]
+        if date_time[0] < 1980:
+            raise ValueError('ZIP does not support timestamps before 1980')
+
     archive_dir = os.path.dirname(zip_filename)
 
     if archive_dir and not os.path.exists(archive_dir):
@@ -217,7 +232,7 @@ def write(self, filename, arcname=None,
             logger.info("adding directory '%s'", base_dir)
             for path in emit_dir_files(base_dir):
                 logger.info("adding '%s'", path)
-                write(zf, path, path)
+                write(zf, path, path, date_time=date_time)
     return zip_filename
 
 
@@ -600,7 +615,7 @@ def zip_cmd(args):
     p.add_argument('zipfile', help='Path to a zip file')
     p.add_argument('dir', nargs=argparse.ONE_OR_MORE,
                    help='Path to a directory for packaging')
-    p.add_argument('-t', '--timestamp', default=timestamp_now_ns(), type=int,
+    p.add_argument('-t', '--timestamp', type=int,
                    help='A timestamp to override for all zip members')
 
 

From 0543d52ff217f4263528e93318a9c43c06cb6bec Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 17:56:06 +0300
Subject: [PATCH 08/21] package.py - Use new zip making implementation in the
 build command

---
 package.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/package.py b/package.py
index 74c97f02..af2e6203 100644
--- a/package.py
+++ b/package.py
@@ -230,9 +230,10 @@ def str_int_to_timestamp(s):
     with zipfile.ZipFile(zip_filename, "w", compression) as zf:
         for base_dir in base_dirs:
             logger.info("adding directory '%s'", base_dir)
-            for path in emit_dir_files(base_dir):
-                logger.info("adding '%s'", path)
-                write(zf, path, path, date_time=date_time)
+            with cd(base_dir):
+                for path in emit_dir_files('.'):
+                    logger.info("adding '%s'", path)
+                    write(zf, path, path, date_time=date_time)
     return zip_filename
 
 
@@ -477,12 +478,7 @@ def create_zip_file(source_dir, target_file):
         target_dir = os.path.dirname(target_file)
         if not os.path.exists(target_dir):
             os.makedirs(target_dir)
-        target_base, _ = os.path.splitext(target_file)
-        shutil.make_archive(
-            target_base,
-            format='zip',
-            root_dir=source_dir,
-        )
+        make_zipfile(target_file, source_dir)
 
     args.dump_env and dump_env('build_command')
 

From 78e44af6fb106bb09b3939abaa8fe97df467606f Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 20:49:00 +0300
Subject: [PATCH 09/21] package.py - Adapted files emitter to produce a zip
 tool similar order

The order will be a dir itself then all dir's files folowed by sub dirs.

It looks like:
  dir1/            - A dir itself
  dir1/file1       - All dir's files
  dir1/file2
  dir1/dir2/       - A dir's sub dir after files
  dir1/dir2/file1
  dir1/dir2/file2

An example may be discovered by:
  > python package.py zip foo.zip examples/fixtures
---
 package.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/package.py b/package.py
index af2e6203..788f6816 100644
--- a/package.py
+++ b/package.py
@@ -124,15 +124,11 @@ def timestamp_now_ns():
 # Packaging functions
 
 def emit_dir_files(base_dir):
-    path = os.path.normpath(base_dir)
-    if path != os.curdir:
-        yield path
-    for dirpath, dirnames, filenames in os.walk(base_dir):
-        for name in sorted(dirnames):
-            path = os.path.normpath(os.path.join(dirpath, name))
-            yield path
-        for name in filenames:
-            path = os.path.normpath(os.path.join(dirpath, name))
+    for root, dirs, files in os.walk(base_dir):
+        if root != '.':
+            yield os.path.normpath(root)
+        for name in files:
+            path = os.path.normpath(os.path.join(root, name))
             if os.path.isfile(path):
                 yield path
 

From d0f33d7f2d3fd309de2a05ebbe57bb91b5089637 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 22:15:15 +0300
Subject: [PATCH 10/21] package.py - Added separate cmd_logger

---
 package.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/package.py b/package.py
index 788f6816..22cc5c4e 100644
--- a/package.py
+++ b/package.py
@@ -35,6 +35,12 @@ def formatMessage(self, record):
 logging.basicConfig(level=logging.INFO, handlers=(log_handler,))
 logger = logging.getLogger()
 
+cmd_logger = logging.getLogger('cmd')
+cmd_log_handler = logging.StreamHandler()
+cmd_log_handler.setFormatter(StderrLogFormatter("> %(message)s"))
+cmd_logger.addHandler(cmd_log_handler)
+cmd_logger.propagate = False
+
 
 ################################################################################
 # Debug helpers
@@ -67,10 +73,11 @@ def abort(message):
 
 
 @contextmanager
-def cd(path):
+def cd(path, silent=False):
     """Changes the working directory."""
     cwd = os.getcwd()
-    logger.info('cd %s', shlex.quote(path))
+    if not silent:
+        cmd_logger.info('cd %s', shlex.quote(path))
     try:
         os.chdir(path)
         yield
@@ -83,7 +90,7 @@ def tempdir():
     """Creates a temporary directory and then deletes it afterwards."""
     prefix = 'terraform-aws-lambda-'
     path = tempfile.mkdtemp(prefix=prefix)
-    logger.info('mktemp -d %sXXXXXXXX # %s', prefix, shlex.quote(path))
+    cmd_logger.info('mktemp -d %sXXXXXXXX # %s', prefix, shlex.quote(path))
     try:
         yield path
     finally:
@@ -225,8 +232,8 @@ def str_int_to_timestamp(s):
 
     with zipfile.ZipFile(zip_filename, "w", compression) as zf:
         for base_dir in base_dirs:
-            logger.info("adding directory '%s'", base_dir)
-            with cd(base_dir):
+            logger.info("adding content of directory '%s'", base_dir)
+            with cd(base_dir, silent=True):
                 for path in emit_dir_files('.'):
                     logger.info("adding '%s'", path)
                     write(zf, path, path, date_time=date_time)
@@ -519,10 +526,11 @@ def create_zip_file(source_dir, target_file):
                 target_path = os.path.join(temp_dir, file_name)
                 target_dir = os.path.dirname(target_path)
                 if not os.path.exists(target_dir):
-                    logger.info('mkdir -p %s', shlex.quote(target_dir))
+                    cmd_logger.info('mkdir -p %s', shlex.quote(target_dir))
                     os.makedirs(target_dir)
-                logger.info('cp %s %s', shlex.quote(file_name),
-                                        shlex.quote(target_path))
+                cmd_logger.info('cp -t %s %s',
+                                shlex.quote(target_dir),
+                                shlex.quote(file_name))
                 shutil.copyfile(file_name, target_path)
                 shutil.copymode(file_name, target_path)
                 shutil.copystat(file_name, target_path)
@@ -562,7 +570,7 @@ def create_zip_file(source_dir, target_file):
                             pip_cache_dir=pip_cache_dir
                         ))
                     else:
-                        logger.info(pip_command)
+                        cmd_logger.info(shlex_join(pip_command))
                         log_handler.flush()
                         check_call(pip_command)
 

From 59738942aec05b09bb90da8b94db07a114fa962c Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 22:17:39 +0300
Subject: [PATCH 11/21] package.py - Fixed file formatting

---
 package.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/package.py b/package.py
index 22cc5c4e..93d45135 100644
--- a/package.py
+++ b/package.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 
 import sys
+
 if sys.version_info < (3, 7):
     raise RuntimeError("A python version 3.7 or newer is required")
 
@@ -25,7 +26,7 @@
 
 class StderrLogFormatter(logging.Formatter):
     def formatMessage(self, record):
-        self._style._fmt = self._style.default_format\
+        self._style._fmt = self._style.default_format \
             if record.name == 'root' else self._fmt
         return super().formatMessage(record)
 
@@ -189,7 +190,7 @@ def write(self, filename, arcname=None,
                     self.fp.seek(self.start_dir)
                 zinfo.header_offset = self.fp.tell()  # Start of header bytes
                 if zinfo.compress_type == zipfile.ZIP_LZMA:
-                # Compressed data includes an end-of-stream (EOS) marker
+                    # Compressed data includes an end-of-stream (EOS) marker
                     zinfo.flag_bits |= 0x02
 
                 self._writecheck(zinfo)
@@ -201,7 +202,7 @@ def write(self, filename, arcname=None,
                 self.start_dir = self.fp.tell()
         else:
             with open(filename, "rb") as src, self.open(zinfo, 'w') as dest:
-                shutil.copyfileobj(src, dest, 1024*8)
+                shutil.copyfileobj(src, dest, 1024 * 8)
 
     def str_int_to_timestamp(s):
         return int(s) / 10 ** (len(s) - 10)

From 5694e18e7d200254f10b2aaee074e094aaae8370 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Tue, 9 Jun 2020 23:48:32 +0300
Subject: [PATCH 12/21] package.py - Improved str_int_to_timestamp
 implementation

---
 package.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/package.py b/package.py
index 93d45135..2b488374 100644
--- a/package.py
+++ b/package.py
@@ -205,7 +205,14 @@ def write(self, filename, arcname=None,
                 shutil.copyfileobj(src, dest, 1024 * 8)
 
     def str_int_to_timestamp(s):
-        return int(s) / 10 ** (len(s) - 10)
+        min_zip_ts = datetime.datetime(1980, 1, 1).timestamp()
+        ts = int(s)
+        if ts < min_zip_ts:
+            return min_zip_ts
+        deg = len(str(int(s))) - 9
+        if deg < 0:
+            ts = ts * 10 ** deg
+        return ts
 
     logger = logging.getLogger('zip')
 

From a9119799ec1cc1b9807d177daf6493e1a74df2d5 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 00:08:07 +0300
Subject: [PATCH 13/21] package.py - Added source code hash output on DEBUG log
 level

---
 package.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/package.py b/package.py
index 2b488374..5b7562ee 100644
--- a/package.py
+++ b/package.py
@@ -18,6 +18,7 @@
 import subprocess
 from subprocess import check_call, call
 from contextlib import contextmanager
+from base64 import b64encode
 import logging
 
 
@@ -128,6 +129,10 @@ def timestamp_now_ns():
     return timestamp
 
 
+def source_code_hash(bytes):
+    return b64encode(hashlib.sha256(bytes).digest()).decode()
+
+
 ################################################################################
 # Packaging functions
 
@@ -587,6 +592,9 @@ def create_zip_file(source_dir, target_file):
         create_zip_file(temp_dir, filename)
         os.utime(filename, ns=(timestamp, timestamp))
         logger.info('Created: %s', shlex.quote(filename))
+        if logger.level <= logging.DEBUG:
+            with open(filename, 'rb') as f:
+                logger.info('Base64sha256: %s', source_code_hash(f.read()))
 
 
 def add_hidden_commands(sub_parsers):

From 4b08675f7a5fc697d39171c52e372974faa8a806 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 00:11:24 +0300
Subject: [PATCH 14/21] package.py - Added source code hash output to a hidden
 zip command

---
 package.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/package.py b/package.py
index 5b7562ee..2c48248b 100644
--- a/package.py
+++ b/package.py
@@ -626,6 +626,10 @@ def zip_cmd(args):
         make_zipfile(args.zipfile, *args.dir, timestamp=args.timestamp)
         logger.info('-' * 80)
         subprocess.call(['zipinfo', args.zipfile])
+        logger.info('-' * 80)
+        logger.info('Source code hash: %s',
+                    source_code_hash(open(args.zipfile, 'rb').read()))
+
     p = hidden_parser('zip', help='Zip folder with provided files timestamp')
     p.set_defaults(command=zip_cmd)
     p.add_argument('zipfile', help='Path to a zip file')

From 8caae55639f4816dbb9160c7354f8b2900049b4b Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 01:52:57 +0300
Subject: [PATCH 15/21] package.py - Applied an earliest possible ts to all zip
 content files as a temporary measure

---
 package.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.py b/package.py
index 2c48248b..3f9230f1 100644
--- a/package.py
+++ b/package.py
@@ -486,7 +486,7 @@ def list_files(top_path):
         results.sort()
         return results
 
-    def create_zip_file(source_dir, target_file):
+    def create_zip_file(source_dir, target_file, timestamp):
         """
         Creates a zip file from a directory.
         """
@@ -494,7 +494,7 @@ def create_zip_file(source_dir, target_file):
         target_dir = os.path.dirname(target_file)
         if not os.path.exists(target_dir):
             os.makedirs(target_dir)
-        make_zipfile(target_file, source_dir)
+        make_zipfile(target_file, source_dir, timestamp=timestamp)
 
     args.dump_env and dump_env('build_command')
 
@@ -589,7 +589,7 @@ def create_zip_file(source_dir, target_file):
 
         # Zip up the temporary directory and write it to the target filename.
         # This will be used by the Lambda function as the source code package.
-        create_zip_file(temp_dir, filename)
+        create_zip_file(temp_dir, filename, timestamp=0)
         os.utime(filename, ns=(timestamp, timestamp))
         logger.info('Created: %s', shlex.quote(filename))
         if logger.level <= logging.DEBUG:

From b12803bd52a1cc03559bd9c59b01525e07c63633 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 01:56:28 +0300
Subject: [PATCH 16/21] package.py - Updated content hashing stuff

---
 package.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/package.py b/package.py
index 3f9230f1..35c251bb 100644
--- a/package.py
+++ b/package.py
@@ -352,24 +352,24 @@ def list_files(top_path):
         results.sort()
         return results
 
-    def generate_content_hash(source_paths):
+    def generate_content_hash(source_paths, hash_func=hashlib.sha256):
         """
         Generate a content hash of the source paths.
         """
 
-        sha256 = hashlib.sha256()
+        hash_obj = hash_func()
 
         for source_path in source_paths:
             if os.path.isdir(source_path):
                 source_dir = source_path
                 for source_file in list_files(source_dir):
-                    update_hash(sha256, source_dir, source_file)
+                    update_hash(hash_obj, source_dir, source_file)
             else:
                 source_dir = os.path.dirname(source_path)
                 source_file = source_path
-                update_hash(sha256, source_dir, source_file)
+                update_hash(hash_obj, source_dir, source_file)
 
-        return sha256
+        return hash_obj
 
     def update_hash(hash_obj, file_root, file_path):
         """
@@ -381,7 +381,7 @@ def update_hash(hash_obj, file_root, file_path):
 
         with open(file_path, 'rb') as open_file:
             while True:
-                data = open_file.read(1024)
+                data = open_file.read(1024 * 8)
                 if not data:
                     break
                 hash_obj.update(data)

From d758af7a05e1fc93a1a95b65db84cb97bd5ab7e0 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 01:57:55 +0300
Subject: [PATCH 17/21] package.py - Fixed unreproducible zip compression for
 python packaging

The cause is that 'pip install' makes wheels unpacking to random
temporary folders and it seems compiles all *.py files also inside of
the temp folders. And each *.pyc file contains a reference
to its original source code file for some reason.
---
 package.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/package.py b/package.py
index 35c251bb..ec4789bd 100644
--- a/package.py
+++ b/package.py
@@ -558,9 +558,8 @@ def create_zip_file(source_dir, target_file, timestamp):
                     else:
                         pip_command = ['pip2']
                     pip_command.extend([
-                        'install',
-                        '--prefix=',
-                        '--target=.',
+                        'install', '--no-compile',
+                        '--prefix=', '--target=.',
                         '--requirement=requirements.txt',
                     ])
                     if docker:

From e5c45ea123c5317e5df2960b48e4ef4e7b0b950c Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 08:45:32 +0300
Subject: [PATCH 18/21] package.py - Improved logging

---
 package.py | 48 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 13 deletions(-)

diff --git a/package.py b/package.py
index ec4789bd..249d6adc 100644
--- a/package.py
+++ b/package.py
@@ -25,23 +25,29 @@
 ################################################################################
 # Logging
 
-class StderrLogFormatter(logging.Formatter):
+class LogFormatter(logging.Formatter):
+    default_format = '%(message)s'
+    formats = {
+        'root': default_format,
+        'build': default_format,
+        'prepare': default_format,
+        'cmd': '> %(message)s',
+        '': '%(name)s: %(message)s'
+    }
+
     def formatMessage(self, record):
-        self._style._fmt = self._style.default_format \
-            if record.name == 'root' else self._fmt
+        self._style._fmt = self.formats.get(record.name, self.formats[''])
         return super().formatMessage(record)
 
 
 log_handler = logging.StreamHandler()
-log_handler.setFormatter(StderrLogFormatter("%(name)s: %(message)s"))
-logging.basicConfig(level=logging.INFO, handlers=(log_handler,))
+log_handler.setFormatter(LogFormatter())
+
 logger = logging.getLogger()
+logger.addHandler(log_handler)
+logger.setLevel(logging.INFO)
 
 cmd_logger = logging.getLogger('cmd')
-cmd_log_handler = logging.StreamHandler()
-cmd_log_handler.setFormatter(StderrLogFormatter("> %(message)s"))
-cmd_logger.addHandler(cmd_log_handler)
-cmd_logger.propagate = False
 
 
 ################################################################################
@@ -265,7 +271,7 @@ def docker_build_command(build_root, docker_file=None, tag=None):
         docker_cmd.extend(['--tag', tag])
     docker_cmd.append(build_root)
 
-    logger.info(shlex_join(docker_cmd))
+    cmd_logger.info(shlex_join(docker_cmd))
     log_handler.flush()
     return docker_cmd
 
@@ -322,7 +328,7 @@ def docker_run_command(build_root, command, runtime,
         docker_cmd.extend([shell, '-c'])
     docker_cmd.extend(command)
 
-    logger.info(shlex_join(docker_cmd))
+    cmd_logger.info(shlex_join(docker_cmd))
     log_handler.flush()
     return docker_cmd
 
@@ -338,16 +344,22 @@ def prepare_command(args):
     Outputs a filename and a command to run if the archive needs to be built.
     """
 
+    logger = logging.getLogger('prepare')
+
     def list_files(top_path):
         """
         Returns a sorted list of all files in a directory.
         """
 
+        _logger = logger.getChild('ls')
+
         results = []
 
         for root, dirs, files in os.walk(top_path):
             for file_name in files:
-                results.append(os.path.join(root, file_name))
+                file_path = os.path.join(root, file_name)
+                results.append(file_path)
+                _logger.debug(file_path)
 
         results.sort()
         return results
@@ -470,11 +482,15 @@ def build_command(args):
     Installs dependencies with pip automatically.
     """
 
+    logger = logging.getLogger('build')
+
     def list_files(top_path):
         """
         Returns a sorted list of all files in a directory.
         """
 
+        _logger = logger.getChild('ls')
+
         results = []
 
         for root, dirs, files in os.walk(top_path):
@@ -482,6 +498,7 @@ def list_files(top_path):
                 file_path = os.path.join(root, file_name)
                 relative_path = os.path.relpath(file_path, top_path)
                 results.append(relative_path)
+                _logger.debug(relative_path)
 
         results.sort()
         return results
@@ -591,7 +608,7 @@ def create_zip_file(source_dir, target_file, timestamp):
         create_zip_file(temp_dir, filename, timestamp=0)
         os.utime(filename, ns=(timestamp, timestamp))
         logger.info('Created: %s', shlex.quote(filename))
-        if logger.level <= logging.DEBUG:
+        if logger.isEnabledFor(logging.DEBUG):
             with open(filename, 'rb') as f:
                 logger.info('Base64sha256: %s', source_code_hash(f.read()))
 
@@ -661,11 +678,16 @@ def args_parser():
 
 def main():
     ns = argparse.Namespace(
+        log_level=os.environ.get('TF_PACKAGE_LOG_LEVEL', 'INFO'),
         dump_input=bool(os.environ.get('TF_DUMP_INPUT')),
         dump_env=bool(os.environ.get('TF_DUMP_ENV')),
     )
     p = args_parser()
     args = p.parse_args(namespace=ns)
+
+    if args.log_level and logging._checkLevel(args.log_level):
+        logging.root.setLevel(args.log_level)
+
     exit(args.command(args))
 
 

From d7cca75d6eb1f00263d72bcac1ef7a2b1c0b7933 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 08:46:01 +0300
Subject: [PATCH 19/21] package.py - Added a -v option to a hidden zip command

---
 package.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/package.py b/package.py
index 249d6adc..3b2c48f7 100644
--- a/package.py
+++ b/package.py
@@ -639,12 +639,15 @@ def hidden_parser(name, **kwargs):
                    nargs=argparse.OPTIONAL)
 
     def zip_cmd(args):
+        if args.verbose:
+            logger.setLevel(logging.DEBUG)
         make_zipfile(args.zipfile, *args.dir, timestamp=args.timestamp)
-        logger.info('-' * 80)
-        subprocess.call(['zipinfo', args.zipfile])
-        logger.info('-' * 80)
-        logger.info('Source code hash: %s',
-                    source_code_hash(open(args.zipfile, 'rb').read()))
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug('-' * 80)
+            subprocess.call(['zipinfo', args.zipfile])
+            logger.debug('-' * 80)
+            logger.debug('Source code hash: %s',
+                         source_code_hash(open(args.zipfile, 'rb').read()))
 
     p = hidden_parser('zip', help='Zip folder with provided files timestamp')
     p.set_defaults(command=zip_cmd)
@@ -653,6 +656,7 @@ def zip_cmd(args):
                    help='Path to a directory for packaging')
     p.add_argument('-t', '--timestamp', type=int,
                    help='A timestamp to override for all zip members')
+    p.add_argument('-v', '--verbose', action='store_true')
 
 
 def args_parser():

From 433efb38ca78f12914f816b4751152d46b1135c5 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 08:51:05 +0300
Subject: [PATCH 20/21] package.py - unify imports usage

---
 package.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.py b/package.py
index 3b2c48f7..38332634 100644
--- a/package.py
+++ b/package.py
@@ -16,7 +16,7 @@
 import tempfile
 import platform
 import subprocess
-from subprocess import check_call, call
+from subprocess import check_call
 from contextlib import contextmanager
 from base64 import b64encode
 import logging
@@ -622,7 +622,7 @@ def hidden_parser(name, **kwargs):
         return p
 
     p = hidden_parser('docker', help='Run docker build')
-    p.set_defaults(command=lambda args: call(docker_run_command(
+    p.set_defaults(command=lambda args: subprocess.call(docker_run_command(
         args.build_root, args.docker_command, args.runtime, interactive=True)))
     p.add_argument('build_root', help='A docker build root folder')
     p.add_argument('docker_command', help='A docker container command',
@@ -631,7 +631,7 @@ def hidden_parser(name, **kwargs):
                    default='python3.8')
 
     p = hidden_parser('docker-image', help='Run docker build')
-    p.set_defaults(command=lambda args: call(docker_build_command(
+    p.set_defaults(command=lambda args: subprocess.call(docker_build_command(
         args.build_root, args.docker_file, args.tag)))
     p.add_argument('-t', '--tag', help='A docker image tag')
     p.add_argument('build_root', help='A docker build root folder')

From cb3d6ff887a0b39fd5e636b15353b5e5342880b7 Mon Sep 17 00:00:00 2001
From: Andrew Hlynskyi <ahlincq@gmail.com>
Date: Wed, 10 Jun 2020 08:51:34 +0300
Subject: [PATCH 21/21] package.py - check zipinfo is available before call it

---
 package.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/package.py b/package.py
index 38332634..73c2df52 100644
--- a/package.py
+++ b/package.py
@@ -643,8 +643,10 @@ def zip_cmd(args):
             logger.setLevel(logging.DEBUG)
         make_zipfile(args.zipfile, *args.dir, timestamp=args.timestamp)
         if logger.isEnabledFor(logging.DEBUG):
-            logger.debug('-' * 80)
-            subprocess.call(['zipinfo', args.zipfile])
+            zipinfo = shutil.which('zipinfo')
+            if zipinfo:
+                logger.debug('-' * 80)
+                subprocess.call([zipinfo, args.zipfile])
             logger.debug('-' * 80)
             logger.debug('Source code hash: %s',
                          source_code_hash(open(args.zipfile, 'rb').read()))