From a3ddfb44d1729dbff37e4998fe45b6dd0baba8d5 Mon Sep 17 00:00:00 2001 From: James Long Date: Wed, 8 Apr 2020 23:57:47 +0000 Subject: [PATCH] Updates to the efficientnet conversion code. * Updated the downloader to download src files that have been added since this package was last updated. * Updated load_efficientnet.py to be tensorflow 1.x/2.x compatible. * tensorflow 1.14+/2.x is now required, as the underlying tensorflow model code now references modules that did not exist in tf 1.12/1.13 * Added a --noweights_only option to the conversion script, which passes through to load_efficientnet to produce .h5 files with both configs and weights. * Added a --parallel_conversion option to the conversion script, which runs the conversion jobs as background processes, and then sequentially waits on them. The conversion job is largely CPU bound, and runs on a single CPU. * Updated requirements to include keras and tensorflow, as they are both required to run the conversion. * Updated README.md * Instructions for installing aria2 * Fixed instructions for the script --- README.md | 37 ++++- requirements.txt | 2 + scripts/convert_from_tf_to_keras.sh | 233 +++++++++++++++++----------- scripts/load_efficientnet.py | 6 +- 4 files changed, 182 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index f29e335..7b054d1 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,38 @@ The performance of each model variant using the pre-trained weights converted fr ## Installation -### Requirements +### Python Requirements -* `Keras >= 2.2.0` / `TensorFlow >= 1.12.0` +* `TensorFlow >= 1.14.0` - as the current versions of efficientnet rely on pacakges + not avaiable in previous releases. +* `Keras >= 2.2.0` * `keras_applications >= 1.0.7` * `scikit-image` +### Other Requirements + +* `aria2` + +For Ubuntu/Linux Mint/Debian Based Systems: + +```bash +sudo apt-get install aria2 +``` + +For CentOS/Fedora Systems: + +```bash +sudo yum install aria2 +``` + +For Manjaro/Arch Systems: +```bash +$ sudo pacman -Sy aria2 +``` + +Or find the most recent releases on the aria2 github repository: +https://github.com/aria2/aria2/releases + ### Installing from the source ```bash @@ -121,10 +147,13 @@ $ pip install -U --pre efficientnet Pick the target directory (like `dist`) and run the [converter script](./scripts) from the repo directory as follows: ```bash -$ ./scripts/convert_efficientnet.sh --target_dir dist +$ ./scripts/convert_from_tf_to_keras.sh --target_dir dist ``` -You can also optionally create the virtual environment with all the dependencies installed by adding `--make_venv=true` and operate in a self-destructing temporary location instead of the target directory by setting `--tmp_working_dir=true`. +You can also optionally: +* Create a virtual environment with all the dependencies installed by adding `--make_venv=true` +* Operate in a self-destructing temporary location instead of the target directory by setting `--tmp_working_dir=true` +* Create a .h5 model file containing both configuration and weights by adding "false" ## Acknowledgements I would like to thanks community members who actively contribute to this repository: diff --git a/requirements.txt b/requirements.txt index f7e99e0..6a35b70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ +tensorflow>=1.14 +keras>=2.20 keras_applications>=1.0.7,<=1.0.8 scikit-image diff --git a/scripts/convert_from_tf_to_keras.sh b/scripts/convert_from_tf_to_keras.sh index 6a807c2..99c1bc1 100755 --- a/scripts/convert_from_tf_to_keras.sh +++ b/scripts/convert_from_tf_to_keras.sh @@ -58,11 +58,13 @@ elementIn() { usage() { echo "Usage:" echo " $0 [ --help | -h ]" - echo " $0 [ --target_dir= | --target_dir ] [options]" + echo " $0 [ --target_dir= | --target_dir ] [--noweights_only] [options]" echo echo "Options:" echo " --use_venv=true|yes|1|t|y:: Use the virtual env with pre-installed dependencies" - echo " --tmp_working_dir=true|yes|1|t|y :: Make a temporary working directory the working space" + echo " --tmp_working_dir= :: Make a temporary working directory the working space" + echo " --noweights_only :: Create model files containing both configuration and weights." + echo " --parallel_conversion :: Run the model conversions in parallel in separate processes" echo echo "The default target_dir is dist." } @@ -71,6 +73,8 @@ usage() { LAST_ARG_IDX=$(($# + 1)) TARGET_DIR="dist" USE_VENV="false" +WEIGHTS_ONLY="true" +PARALLEL_CONVERSION="false" declare -A LONGOPTS # Use associative array to declare how many arguments a long option # expects. In this case we declare that loglevel expects/has one @@ -80,75 +84,91 @@ LONGOPTS=([target_dir]=1 [use_venv]=1) OPTSPEC="h-:" while getopts "$OPTSPEC" opt; do while true; do - case "${opt}" in - -) #OPTARG is name-of-long-option or name-of-long-option=value - if [[ ${OPTARG} =~ .*=.* ]]; then # with this --key=value format only one argument is possible - opt=${OPTARG/=*/} - ((${#opt} <= 1)) && { - echo "Syntax error: Invalid long option '$opt'" >&2 - exit 2 - } - if (($((LONGOPTS[$opt])) != 1)); then - echo "Syntax error: Option '$opt' does not support this syntax." >&2 - exit 2 - fi - OPTARG=${OPTARG#*=} - else #with this --key value1 value2 format multiple arguments are possible - opt="$OPTARG" - ((${#opt} <= 1)) && { - echo "Syntax error: Invalid long option '$opt'" >&2 - exit 2 - } - OPTARG=(${@:OPTIND:$((LONGOPTS[$opt]))}) - ((OPTIND += LONGOPTS[$opt])) - echo $OPTIND - ((OPTIND > $LAST_ARG_IDX)) && { - echo "Syntax error: Not all required arguments for option '$opt' are given." >&2 - exit 3 - } - fi - continue - ;; - target_dir) - TARGET_DIR=$(realpath $OPTARG) - ;; - use_venv) - USE_VENV=$OPTARG - ;; - tmp_working_dir) - MAKE_TMP_WORKING_DIR=$OPTARG - ;; - h | help) - usage - exit 0 - ;; - ?) - echo "Syntax error: Unknown short option '$OPTARG'" >&2 - exit 2 - ;; - *) - echo "Syntax error: Unknown long option '$opt'" >&2 - exit 2 - ;; - esac - break + case "${opt}" in + -) #OPTARG is name-of-long-option or name-of-long-option=value + if [[ ${OPTARG} =~ .*=.* ]]; then # with this --key=value format only one argument is possible + opt=${OPTARG/=*/} + ((${#opt} <= 1)) && { + echo "Syntax error: Invalid long option '$opt'" >&2 + exit 2 + } + if (($((LONGOPTS[$opt])) != 1)); then + echo "Syntax error: Option '$opt' does not support this syntax." >&2 + exit 2 + fi + OPTARG=${OPTARG#*=} + else #with this --key value1 value2 format multiple arguments are possible + opt="$OPTARG" + ((${#opt} <= 1)) && { + echo "Syntax error: Invalid long option '$opt'" >&2 + exit 2 + } + OPTARG=(${@:OPTIND:$((LONGOPTS[$opt]))}) + ((OPTIND += LONGOPTS[$opt])) + echo $OPTIND + ((OPTIND > $LAST_ARG_IDX)) && { + echo "Syntax error: Not all required arguments for option '$opt' are given." >&2 + exit 3 + } + fi + continue + ;; + target_dir) + TARGET_DIR=$(realpath $OPTARG) + ;; + use_venv) + USE_VENV=$OPTARG + ;; + tmp_working_dir) + MAKE_TMP_WORKING_DIR=$OPTARG + ;; + noweights_only) + WEIGHTS_ONLY="false" + ;; + parallel_conversion) + PARALLEL_CONVERSION="true" + ;; + h | help) + usage + exit 0 + ;; + ?) + echo "Syntax error: Unknown short option '$OPTARG'" >&2 + exit 2 + ;; + *) + echo "Syntax error: Unknown long option '$opt'" >&2 + exit 2 + ;; + esac + break done done # internal variables SOURCE_CODE_URL="https://raw.githubusercontent.com/tensorflow/tpu/master/models/official/efficientnet/" SOURCE_CODE_FILES=( + "condconv/__init__.py" + "condconv/condconv_layers.py" + "condconv/efficientnet_condconv_builder.py" + "edgetpu/__init__.py" + "edgetpu/efficientnet_edgetpu_builder.py" "efficientnet_builder.py" "efficientnet_model.py" "eval_ckpt_main.py" + "lite/__init__.py" + "lite/efficientnet_lite_builder.py" + "model_builder_factory.py" "utils.py" "preprocessing.py" + "tpu/__init__.py" + "tpu/efficientnet_tpu_builder.py" ) SOURCE_CODE_DIR="tf_src" CHECKPOINTS_DIR="pretrained_tensorflow" CHECKPOINT_PREFIX="efficientnet-" -CHECKPOINTS_URL="https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/" +CHECKPOINTS_URL="https://storage.googleapis.com/cloud-tpu-checkpoints/efficientnet/ckpts/" CHECKPOINTS_EXT=".tar.gz" CONVERTED_MODELS_DIR="pretrained_keras" @@ -191,11 +211,11 @@ cd $WORKING_DIR if elementIn "$USE_VENV" "${TRUE[@]}"; then if [ -d $VIRTUALENV_DIR ]; then - source $VIRTUALENV_DIR/bin/activate + source $VIRTUALENV_DIR/bin/activate else - virtualenv --no-site-packages $VIRTUALENV_DIR - source $VIRTUALENV_DIR/bin/activate - pip install tensorflowjs keras scikit-image + virtualenv --no-site-packages $VIRTUALENV_DIR + source $VIRTUALENV_DIR/bin/activate + pip install tensorflowjs keras scikit-image fi fi @@ -203,20 +223,20 @@ if ! [ -d $CHECKPOINTS_DIR ]; then notify "=" "Downloading the checkpoints..." mkdir -p $CHECKPOINTS_DIR for MODEL_VERSION in "${MODELS[@]}"; do - if ! [ -d $CHECKPOINT_PREFIX$MODEL_VERSION ]; then - cd $CHECKPOINTS_DIR - # -x: maximum number of connections per server - # -k: minimum split size for multi-source download - # -o: target filename - aria2c \ - -x 16 \ - -k 1M \ - -o $MODEL_VERSION$CHECKPOINTS_EXT \ - $CHECKPOINTS_URL$CHECKPOINT_PREFIX$MODEL_VERSION$CHECKPOINTS_EXT - tar xvf $MODEL_VERSION$CHECKPOINTS_EXT - rm $MODEL_VERSION$CHECKPOINTS_EXT - cd .. - fi + if ! [ -d $CHECKPOINT_PREFIX$MODEL_VERSION ]; then + cd $CHECKPOINTS_DIR + # -x: maximum number of connections per server + # -k: minimum split size for multi-source download + # -o: target filename + aria2c \ + -x 16 \ + -k 1M \ + -o $MODEL_VERSION$CHECKPOINTS_EXT \ + $CHECKPOINTS_URL$CHECKPOINT_PREFIX$MODEL_VERSION$CHECKPOINTS_EXT + tar xvf $MODEL_VERSION$CHECKPOINTS_EXT + rm $MODEL_VERSION$CHECKPOINTS_EXT + cd .. + fi done fi @@ -227,30 +247,65 @@ if ! [ -d $SOURCE_CODE_DIR ]; then mkdir -p $SOURCE_CODE_DIR touch $SOURCE_CODE_DIR/__init__.py for SOURCE_CODE_FILE in "${SOURCE_CODE_FILES[@]}"; do - aria2c -x 16 -k 1M -o $SOURCE_CODE_DIR/$SOURCE_CODE_FILE $SOURCE_CODE_URL$SOURCE_CODE_FILE + aria2c -x 16 -k 1M -o $SOURCE_CODE_DIR/$SOURCE_CODE_FILE $SOURCE_CODE_URL$SOURCE_CODE_FILE done fi + cd $PARENT_DIR cd $TARGET_DIR -for MODEL_VERSION in "${MODELS[@]}"; do - MODEL_NAME="efficientnet-"$MODEL_VERSION +if [ $PARALLEL_CONVERSION == "true" ] +then + PIDS=() + MODEL_NAMES=() + for MODEL_VERSION in "${MODELS[@]}"; do + MODEL_NAME="efficientnet-"$MODEL_VERSION + notify "~" "Converting $MODEL_NAME (see $WORKING_DIR/$MODEL_NAME.[logs|error_logs])" - notify "~" "Converting $MODEL_NAME..." + PYTHONPATH=.. python $SCRIPT_DIR/load_efficientnet.py \ + --model_name $MODEL_NAME \ + --source $SOURCE_CODE_DIR \ + --tf_checkpoint $CHECKPOINTS_DIR/$MODEL_NAME \ + --output_file $CONVERTED_MODELS_DIR/$MODEL_NAME \ + --weights_only $WEIGHTS_ONLY 2>$WORKING_DIR/$MODEL_NAME.error_logs \ + >$WORKING_DIR/$MODEL_NAME.logs & - WEIGHTS_ONLY="true" + PID=$! + PIDS+=($PID) + MODEL_NAMES+=($MODEL_NAME) + done - if ! [ -z "$2" ] && [ "$2" != "true" ]; then - WEIGHTS_ONLY="false" - fi - PYTHONPATH=.. python $SCRIPT_DIR/load_efficientnet.py \ - --model_name $MODEL_NAME \ - --source $SOURCE_CODE_DIR \ - --tf_checkpoint $CHECKPOINTS_DIR/$MODEL_NAME \ - --output_file $CONVERTED_MODELS_DIR/$MODEL_NAME \ - --weights_only $WEIGHTS_ONLY -done + NUM_MODELS=${#MODELS[@]} + for i in `seq 0 $((NUM_MODELS - 1))` + do + MODEL_NAME=${MODEL_NAMES[$i]} + PID=${PIDS[$i]} + + notify "~" "Waiting for $MODEL_NAME to complete (process: $PID)" + wait $PID + if [ $? -eq 0 ] + then + notify "=" "$MODEL_NAME conversion successful!" + else + notify "!" "$MODEL_NAME conversion FAILED - see $WORKING_DIR/$MODEL_NAME.log!" + fi + done -notify "=" "Success!" +else + for MODEL_VERSION in "${MODELS[@]}"; do + + MODEL_NAME="efficientnet-"$MODEL_VERSION + + notify "~" "Converting $MODEL_NAME..." + + PYTHONPATH=.. python $SCRIPT_DIR/load_efficientnet.py \ + --model_name $MODEL_NAME \ + --source $SOURCE_CODE_DIR \ + --tf_checkpoint $CHECKPOINTS_DIR/$MODEL_NAME \ + --output_file $CONVERTED_MODELS_DIR/$MODEL_NAME \ + --weights_only $WEIGHTS_ONLY + done + notify "=" "Success!" +fi diff --git a/scripts/load_efficientnet.py b/scripts/load_efficientnet.py index 67372f1..ec786ba 100644 --- a/scripts/load_efficientnet.py +++ b/scripts/load_efficientnet.py @@ -93,14 +93,14 @@ def convert_tensorflow_model( """ Loads and saves a TensorFlow model. """ image_files = [example_img] eval_ckpt_driver = eval_ckpt_main.EvalCkptDriver(model_name) - with tf.Graph().as_default(), tf.Session() as sess: + with tf.Graph().as_default(), tf.compat.v1.Session() as sess: images, _ = eval_ckpt_driver.build_dataset( image_files, [0] * len(image_files), False ) eval_ckpt_driver.build_model(images, is_training=False) - sess.run(tf.global_variables_initializer()) + sess.run(tf.compat.v1.global_variables_initializer()) eval_ckpt_driver.restore_model(sess, model_ckpt) - global_variables = tf.global_variables() + global_variables = tf.compat.v1.global_variables() weights = dict() for variable in global_variables: try: