fixed bug in fuse_collection

configurations/template_full_montage.json

@@ -0,0 +1,70 @@
+{
+    "__comments": {
+        "comment_0": "IMPORTANT: Before running check: source_collection, target_collection ...",
+        "comment_1": "... target_point_match_collection, scratch and section_number",
+        "comment_2": "Parameters to tweak I : solver_options.scale, solver_options.lambda",
+        "comment_3": "Parameters to tweak II: solver_options.min_points, solver_options.max_points",
+        "comment_4": " common values: solver_options.scale 0.3-0.5, solver_options.lambda = 0.0001 - 100",
+        "comment_5": " common values: solver_options.min_points = 3, solver_options.max_points = 100"
+    },
+	"solver_options": {
+		"min_tiles": 3,
+		"degree": 1,
+		"outlier_lambda": 1000,
+		"solver": "backslash",
+		"min_points": 10,
+        "max_points": 50,
+		"nbrs": 0,
+		"xs_weight": 0.05,
+		"stvec_flag": 0,
+		"conn_comp": 1,
+		"distributed": 0,
+		"lambda": 0.1,
+		"edge_lambda": 0.1,
+		"small_region_lambda": 10,
+		"small_region": 5,
+		"calc_confidence": 1,
+		"translation_fac": 1,
+        "dthresh_factor": 1.2,
+        "scale": 0.5,
+        "use_peg": 1,
+        "peg_weight": 0.0001,
+        "peg_npoints": 5
+	},
+    "SURF_options": {
+		"SURF_NumOctaves": 2,
+		"SURF_NumScaleLevels": 3,
+        "SURF_MetricThreshold": 1000,
+        "SURF_MaxFeatures": 100
+	},
+	"source_collection": {
+		"stack": "v1_acquire",
+		"owner": "flyTEM",
+		"project": "spc",
+		"service_host": "10.40.3.162:8080",
+		"baseURL": "http://10.40.3.162:8080/render-ws/v1",
+		"verbose": 1
+	},
+	"target_collection": {
+		"stack": "v1_montage_SURF_kk04_P2",
+		"owner": "flyTEM",
+		"project": "spc",
+		"service_host": "10.40.3.162:8080",
+		"baseURL": "http://10.40.3.162:8080/render-ws/v1",
+		"verbose": 1,
+        "versionNotes": "experiments to find optimal parameters for Allen dataset"
+	},
+	"target_point_match_collection": {
+		"server": "http://10.40.3.162:8080/render-ws/v1",
+		"owner": "flyTEM",
+		"match_collection": "spc_test"
+	},
+	"section_number": 3357,
+    "image_filter": "true",
+    "scratch": "/gpfs1/scratch/spc/matlab_work/montage/scratch",
+    "renderer_client": "/groups/flyTEM/flyTEM/render/bin/render.sh",
+	"verbose": 2,
+    "ncpus": 8,
+    "complete": 0
+
+}

configurations/template_solve_montage.json

@@ -0,0 +1,49 @@
+{
+	"solver_options": {
+		"min_tiles": 3,
+		"degree": 1,
+		"outlier_lambda": 1000,
+		"solver": "backslash",
+		"min_points": 10,
+        "max_points": 100,
+		"stvec_flag": 0,
+		"conn_comp": 1,
+		"distributed": 0,
+		"lambda": 0.1,
+		"edge_lambda": 0.1,
+		"small_region_lambda": 10,
+		"small_region": 5,
+		"calc_confidence": 1,
+		"translation_fac": 1,
+        "use_peg": 1,
+        "peg_weight": 0.00001,
+        "peg_npoints": 5
+	},
+	"source_collection": {
+		"stack": "v12_acquire_merged",
+		"owner": "flyTEM",
+		"project": "FAFB00",
+		"service_host": "10.37.5.60:8080",
+		"baseURL": "http://10.37.5.60:8080/render-ws/v1",
+		"verbose": 1
+	},
+	"source_point_match_collection": {
+		"server": "http://10.40.3.162:8080/render-ws/v1",
+		"owner": "flyTEM",
+		"match_collection": "FAFB_pm_2"
+	},
+	"target_collection": {
+		"stack": "EXP_test_montage_section_5",
+		"owner": "flyTEM",
+		"project": "test",
+		"service_host": "10.37.5.60:8080",
+		"baseURL": "http://10.37.5.60:8080/render-ws/v1",
+		"verbose": 1,
+        "initialize": 0,
+        "complete": 1
+	},
+	"z_value": 5,
+    "filter_point_matches": 1,
+    "temp_dir":"/scratch/khairyk",
+	"verbose": 1
+}

external/SIFT_point_match_scripts/01_create_stage_stack.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+FROM_OWNER="flyTEM"
+FROM_PROJECT="FAFB00"
+FROM_STACK="v12_acquire_merged"
+
+MIN_Z="1949"
+MAX_Z="1957"
+
+TO_OWNER="flyTEM"
+TO_PROJECT="khairyk_stage"
+TO_STACK="z_${MIN_Z}_${MAX_Z}_acquire"
+
+# ------------------------------------------------------------------------------------
+# you should be able to leave the rest as-is ...
+
+ARGS="--baseDataUrl http://tem-services:8080/render-ws/v1"
+ARGS="${ARGS} --owner ${FROM_OWNER} --project ${FROM_PROJECT} --fromStack ${FROM_STACK}"
+ARGS="${ARGS} --toOwner ${TO_OWNER} --toProject ${TO_PROJECT} --toStack ${TO_STACK}"
+ARGS="${ARGS} --replaceLastTransformWithStage --splitMergedSections"
+ARGS="${ARGS} --completeToStackAfterCopy"
+
+for Z in `seq ${MIN_Z} ${MAX_Z}`; do
+  ARGS="${ARGS} --z ${Z}"
+done
+
+/groups/flyTEM/flyTEM/render/bin/copy-layer.sh ${ARGS}

external/SIFT_point_match_scripts/02_gen_montage_sift_matches.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+ABSOLUTE_SCRIPT=`readlink -m $0`
+SCRIPT_DIR=`dirname ${ABSOLUTE_SCRIPT}`
+
+# ==========================================================================
+# Change these parameters for each run ...
+
+export BILL_TO="flyTEM"
+export JOB_RUN_TIME="1814400" # 86400 seconds per day * 21 days = 1814400 second hard runtime
+
+# source data parameters
+SERVICE_HOST="10.40.3.162:8080"      # use IP address for tem-services until DNS issue is resolved
+MATCH_OWNER="flyTEM"
+MATCH_COLLECTION="FAFB_pm_2"
+
+for JSON_FILE in `ls ${SCRIPT_DIR}/tile_pairs_*dist_0.json*`; do
+  PAIR_JSON_ARGS="${PAIR_JSON_ARGS} --pairJson ${JSON_FILE}"
+done
+
+# Default SIFT parameters are:
+#   --renderWithFilter true 
+#   --renderWithoutMask true 
+#   --renderScale 1.0 
+#   --fillWithNoise true
+#   --SIFTfdSize 8
+#   --SIFTminScale 0.5
+#   --SIFTmaxScale 0.85
+#   --SIFTsteps 3
+SIFT_PARAMETERS="--SIFTminScale 0.48 --renderScale 0.6 --SIFTmaxScale 0.62 --SIFTsteps 3"
+
+# Default match filtering parameters are:
+#   --matchRod 0.92
+#   --matchMaxEpsilon 20.0
+#   --matchMinInlierRatio 0.0
+#   --matchMinNumInliers 10
+MATCH_FILTER_PARAMETERS="--matchRod 0.8 --matchMaxEpsilon 20.0 --matchMinInlierRatio 0.0 --matchMinNumInliers 8 --matchMaxNumInliers 40"
+
+# To be nice to others, avoid requesting more than 60 nodes.
+# When the cluster is busy, you may want to decrease node count to get running since
+# Spark job won't start until number of requested nodes are available.
+NUMBER_OF_SPARK_NODES=20
+
+# ==========================================================================
+# You should be able to leave everything under here as is ...
+
+# ==========================================================================
+# setup output directory
+# (NOTE: you can leave this as is if output can go to the standard location)
+# ==========================================================================
+
+YEAR_MONTH=`date +"%Y%m"`
+DAY=`date +"%d"`
+TIME=`date +"%H%M%S"`
+
+export SPARK_OUTPUT_DIR="/groups/flyTEM/flyTEM/spark/work/${USER}/${YEAR_MONTH}/${DAY}/${TIME}_$$"
+export MASTER_ENV_FILE="${SPARK_OUTPUT_DIR}/master_env.sh"
+
+export LOG_DIR="${SPARK_OUTPUT_DIR}/logs"
+export TMP="${SPARK_OUTPUT_DIR}/tmp"
+
+echo """
+  creating ${SPARK_OUTPUT_DIR}
+"""
+
+mkdir -p ${SPARK_OUTPUT_DIR}
+
+# ==========================================================================
+# build java args and launch spark
+# (NOTE: you should not need to modify anything below)
+# ==========================================================================
+
+echo """
+  launching Spark job
+"""
+
+JAR="/groups/flyTEM/flyTEM/render/lib/current-spark-standalone.jar"
+CLASS="org.janelia.render.client.spark.SIFTPointMatchClient"
+
+ARGV="--baseDataUrl http://${SERVICE_HOST}/render-ws/v1 --owner ${MATCH_OWNER} --collection ${MATCH_COLLECTION}"
+ARGV="${ARGV} ${SIFT_PARAMETERS} ${MATCH_FILTER_PARAMETERS} --maxFeatureCacheGb 15 ${PAIR_JSON_ARGS}"
+
+/groups/flyTEM/flyTEM/render/spark/bin/inflame.sh ${NUMBER_OF_SPARK_NODES} ${JAR} ${CLASS} ${ARGV}
+
+# from /usr/local/spark-current/conf/spark-env.sh
+LIKELY_SPARK_LOG_DIR=~/.spark/logs/$(date +%H-%F)
+
+sleep 2
+
+mkdir -p logs
+SPARK_LAUNCH_LOG="logs/spark_launch.log"
+
+qstat | grep spark | tee -a ${SPARK_LAUNCH_LOG}
+
+echo """
+  cluster logs will be written to ${LOG_DIR}
+
+  internal spark logs will likely be written to ${LIKELY_SPARK_LOG_DIR}
+
+""" | tee -a ${SPARK_LAUNCH_LOG}

external/SIFT_point_match_scripts/02_gen_pairs.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+#--------------------------------------
+# set up global parameters
+#--------------------------------------
+
+OWNER="flyTEM"                            # owner of render stacks and match collections
+LOCATION_PROJECT="test"                 # project of render stack used for tile locations
+LOCATION_STACK="temp_rough_slab"                # render stack used for tile locations
+MIN_Z="102019"                                 # minimum z value for layers to include in potential tile pairs 
+MAX_Z="102022"                               # maximum z value for layers to include in potential tile pairs
+
+RENDER_PROJECT="khairyk_stage"                   # project of render stack used for rendering during point match derivation
+RENDER_STACK="z_1949_1957_acquire"         # render stack used for rendering during point match derivation
+
+# base command for running the tile pair client
+BASE_CMD="/groups/flyTEM/flyTEM/render/pipeline/bin/run_ws_client.sh 6G org.janelia.render.client.TilePairClient"
+LOG_DIR="logs"
+RUN_TIME=`date +"%Y%m%d_%H%M%S"`
+PAIR_GEN_LOG="${LOG_DIR}/tile_pairs-${RUN_TIME}.log"
+
+mkdir -p ${LOG_DIR}
+
+#--------------------------------------
+# generate within-layer potential pairs
+#--------------------------------------
+
+DISTANCE="0"                              # distance in z from each layer to look for potential tile pairs 
+
+# xyNeighborFactor is used to determine radial distance from tile center to look for potential pairs
+FILTER_OPTS="--xyNeighborFactor 0.6 --excludeCornerNeighbors true --excludeSameLayerNeighbors false --excludeCompletelyObscuredTiles true"
+
+P1="--baseDataUrl http://10.40.3.162:8080/render-ws/v1 --owner ${OWNER} --project ${LOCATION_PROJECT}"
+P2="--baseProject ${RENDER_PROJECT} --baseStack ${RENDER_STACK} --stack ${LOCATION_STACK}"
+P3="--minZ ${MIN_Z} --maxZ ${MAX_Z}"
+JSON_FILE="tile_pairs_${LOCATION_STACK}_z_${MIN_Z}_to_${MAX_Z}_dist_${DISTANCE}.json.gz"
+
+${BASE_CMD} ${P1} ${P2} ${P3} ${FILTER_OPTS} --zNeighborDistance ${DISTANCE} --toJson ${JSON_FILE} | tee -a ${PAIR_GEN_LOG}
+
+#--------------------------------------
+# generate outside-layer potential pairs
+#--------------------------------------
+
+# exit 0  # NOTE: exit here since cross layer pairs don't make sense before stack is rough aligned
+
+DISTANCE="4"
+FILTER_OPTS="--xyNeighborFactor 0.6 --excludeCornerNeighbors false --excludeSameLayerNeighbors true --excludeCompletelyObscuredTiles true"
+JSON_FILE="tile_pairs_${LOCATION_STACK}_z_${MIN_Z}_to_${MAX_Z}_dist_${DISTANCE}.json.gz"
+
+${BASE_CMD} ${P1} ${P2} ${P3} ${FILTER_OPTS} --zNeighborDistance ${DISTANCE} --toJson ${JSON_FILE} | tee -a ${PAIR_GEN_LOG}