forked from kubeflow/kubeflow
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Tooling to make it easier to tag images and update the ksonnet protot…
…ypes (kubeflow#1066) * Provide some python scripts to match images and then apply a tag to them like "v0.2.0". This makes it possible to easily apply a new release tag to set of images like the Jupyter images. * Creeate a shell script to use sed and other twos to update images in our ksonnet prototypes. * Add instructions for doing this. * I used the scripts to add the v0.2.0 tag to our Jupyter images. Related to kubeflow#1060
- Loading branch information
1 parent
61adfb4
commit 8366029
Showing
8 changed files
with
510 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| """The script uses a regex to identify images in GCR and add | ||
| entries for them to image_tags.yaml | ||
| """ | ||
|
|
||
| import argparse | ||
| import logging | ||
| import re | ||
| import json | ||
| import yaml | ||
|
|
||
| from kubeflow.testing import util | ||
|
|
||
| def main(unparsed_args=None): # pylint: disable=too-many-locals | ||
| logging.getLogger().setLevel(logging.INFO) # pylint: disable=too-many-locals | ||
| # create the top-level parser | ||
| parser = argparse.ArgumentParser( | ||
| description="Get Images by regex") | ||
|
|
||
| parser.add_argument( | ||
| "--pattern", | ||
| default="", | ||
| type=str, | ||
| help="Regex pattern e.g. .*tensorflow.*notebook.*:v20180619.*") | ||
|
|
||
| parser.add_argument( | ||
| "--images_file", | ||
| default="image_tags.yaml", | ||
| type=str, | ||
| help="Yaml file containing the tags to attach.") | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| with open(args.images_file) as hf: | ||
| config = yaml.load(hf) | ||
|
|
||
| existing_images = {} | ||
|
|
||
| for image in config["images"]: | ||
| existing_images[image["name"]] = {} | ||
| for v in image["versions"]: | ||
| existing_images[image["name"]][v["digest"]] = v | ||
|
|
||
| raw_images = util.run(["gcloud", | ||
| "--project=kubeflow-images-public", | ||
| "container", "images", "list", | ||
| "--format=json"]) | ||
|
|
||
| all_images = json.loads(raw_images) | ||
| name_pattern, tag_pattern = args.pattern.split(":") | ||
|
|
||
| name_re = re.compile(name_pattern) | ||
| tag_re = re.compile(tag_pattern) | ||
|
|
||
| matching = [] | ||
| for image in all_images: | ||
| if not name_re.match(image["name"]): | ||
| continue | ||
| logging.info("Matching image: %s", image["name"]) | ||
| matching.append(image) | ||
|
|
||
| # For each image ist all tags and find the matching ones | ||
| images_to_add = {} | ||
| for image in matching: | ||
| raw_tags = util.run(["gcloud", | ||
| "--project=kubeflow-images-public", | ||
| "container", "images", "list-tags", image["name"], | ||
| "--format=json"]) | ||
|
|
||
| tags = json.loads(raw_tags) | ||
|
|
||
| for info in tags: | ||
| for t in info["tags"]: | ||
| if tag_re.match(t): | ||
| is_match = True | ||
| versions = images_to_add.get(image["name"], {}) | ||
| versions[info["digest"]] = info | ||
| images_to_add[image["name"]] = versions | ||
|
|
||
| # Merge in any missing versions | ||
| for name, versions in images_to_add.iteritems(): | ||
| if not name in existing_images: | ||
| existing_images[name] = {} | ||
|
|
||
| for v in versions.itervalues(): | ||
| if v["digest"] in existing_images[name]: | ||
| logging.info("Image %s sha %s already defined.", name, v["digest"]) | ||
| else: | ||
| logging.info("Image %s adding sha %s", name, v["digest"]) | ||
| existing_images[name][v["digest"]] = v | ||
|
|
||
| # Convert to the expected output | ||
| output = {} | ||
| output["images"] = [] | ||
|
|
||
| names = existing_images.keys() | ||
| names.sort() | ||
| for name in names: | ||
| versions = existing_images[name] | ||
| new_image = {} | ||
| new_image["name"] = name | ||
| new_image["versions"] = [] | ||
| for v in versions.itervalues(): | ||
| new_image["versions"].append(v) | ||
|
|
||
| output["images"].append(new_image) | ||
|
|
||
| with open(args.images_file, "w") as hf: | ||
| hf.write(yaml.safe_dump(output, default_flow_style=False)) | ||
| logging.info("Done.") | ||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig(level=logging.INFO, | ||
| format=('%(levelname)s|%(asctime)s' | ||
| '|%(pathname)s|%(lineno)d| %(message)s'), | ||
| datefmt='%Y-%m-%dT%H:%M:%S', | ||
| ) | ||
| logging.getLogger().setLevel(logging.INFO) | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| """This script adds or moves a tag in image_tags.yaml | ||
| This script doesn't actually update the images. For that you need to | ||
| call apply_image_tags using image_tags.yaml | ||
| The script looks for images matching a regex and will add a tag to that | ||
| image. If that tag is already on an existing version of the image it is removed. | ||
| Example: | ||
| python add_image_tag.py --pattern=.*tensorflow.*1.*notebook.*:v20180619.* \ | ||
| --tag=v0.2.0 | ||
| This would add the tag v0.2.0 to images matching the pattern and remove it | ||
| from any existing images. | ||
| """ | ||
|
|
||
| import argparse | ||
| import logging | ||
| import re | ||
| import yaml | ||
|
|
||
| from kubeflow.testing import util | ||
|
|
||
| def main(unparsed_args=None): # pylint: disable=too-many-locals | ||
| logging.getLogger().setLevel(logging.INFO) # pylint: disable=too-many-locals | ||
| # create the top-level parser | ||
| parser = argparse.ArgumentParser( | ||
| description="Apply tags to file") | ||
|
|
||
| parser.add_argument( | ||
| "--images_file", | ||
| default="image_tags.yaml", | ||
| type=str, | ||
| help="Yaml file containing the tags to attach.") | ||
|
|
||
| parser.add_argument( | ||
| "--pattern", | ||
| default="", | ||
| type=str, | ||
| help=("Regex pattern e.g. .*tensorflow.*notebook.*:v20180619.* " | ||
| "to select the images to apply.")) | ||
|
|
||
| parser.add_argument( | ||
| "--tag", | ||
| default="", | ||
| type=str, | ||
| help="The tag to apply") | ||
|
|
||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| with open(args.images_file) as hf: | ||
| config = yaml.load(hf) | ||
|
|
||
| if not config: | ||
| raise ValueError("No images could be load from %s" % args.images_file) | ||
| name_pattern, tag_pattern = args.pattern.split(":") | ||
| name_re = re.compile(name_pattern) | ||
| tag_re = re.compile(tag_pattern) | ||
|
|
||
| for image in config["images"]: | ||
| name = image["name"] | ||
| if not name_re.match(name): | ||
| continue | ||
|
|
||
| # Loop over all the images and see if the supplied tag is already | ||
| # mapped to an image and which version to add the label to. | ||
| # The index of the version to add the tag to. | ||
| new_index = [] | ||
| existing_index = [] | ||
| for v_index, v in enumerate(image["versions"]): | ||
| for tag in v["tags"]: | ||
| if tag == args.tag: | ||
| existing_index.append(v_index) | ||
|
|
||
| if tag_re.match(tag): | ||
| new_index.append(v_index) | ||
|
|
||
| if len(existing_index) > 1: | ||
| logging.error("Multiple images %s had tag %s", name, args.tag) | ||
|
|
||
| # TODO(jlewi) | ||
| if existing_index and not new_index: | ||
| logging.error("Not moving tag for image %s because no images matched %s", | ||
| name, args.pattern) | ||
| existing_index = [] | ||
| for e in existing_index: | ||
| image["versions"][e]["tags"].remove(args.tag) | ||
|
|
||
| logging.info("Image %s removing tag from sha %s", name, | ||
| image["versions"][e]["digest"]) | ||
|
|
||
| if len(new_index) > 1: | ||
| raise ValueError("Image {0} had {1} images match {2}".format(name, len( | ||
| new_index, args.pattern))) | ||
|
|
||
| if new_index: | ||
| v = image["versions"][new_index[0]] | ||
| logging.info("Image %s adding tag from sha %s", name, | ||
| v["digest"]) | ||
| v["tags"].append(args.tag) | ||
|
|
||
| with open(args.images_file, "w") as hf: | ||
| hf.write(yaml.safe_dump(config, default_flow_style=False)) | ||
| logging.info("Done.") | ||
|
|
||
| if __name__ == "__main__": | ||
| logging.basicConfig(level=logging.INFO, | ||
| format=('%(levelname)s|%(asctime)s' | ||
| '|%(pathname)s|%(lineno)d| %(message)s'), | ||
| datefmt='%Y-%m-%dT%H:%M:%S', | ||
| ) | ||
| logging.getLogger().setLevel(logging.INFO) | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.