From ed91ac414d5c450db6e21816085c9f8e0c2e81e3 Mon Sep 17 00:00:00 2001 From: jingpengw Date: Thu, 29 Oct 2020 14:03:08 -0400 Subject: [PATCH] slurm cluster is running, will see whether it is correct or not --- chunkflow/chunk/image/convnet/inferencer.py | 5 ++++- chunkflow/flow/flow.py | 18 +++++++++--------- chunkflow/plugins/mapto01.py | 3 +++ requirements.txt | 1 - 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/chunkflow/chunk/image/convnet/inferencer.py b/chunkflow/chunk/image/convnet/inferencer.py index 74ffcb3e..94b17bdd 100644 --- a/chunkflow/chunk/image/convnet/inferencer.py +++ b/chunkflow/chunk/image/convnet/inferencer.py @@ -60,7 +60,10 @@ def __init__(self, self.input_size = input_size if output_crop_margin is None: - self.output_crop_margin = self.output_patch_overlap + if mask_output_chunk: + self.output_crop_margin = (0,0,0) + else: + self.output_crop_margin = self.output_patch_overlap else: self.output_crop_margin = output_crop_margin # we should always crop more than the patch overlap diff --git a/chunkflow/flow/flow.py b/chunkflow/flow/flow.py index 8c54afb5..13ab536b 100755 --- a/chunkflow/flow/flow.py +++ b/chunkflow/flow/flow.py @@ -227,7 +227,7 @@ def create_info(tasks,layer_path, channel_num, layer_type, data_type, encoding, type=click.Path(file_okay=True, dir_okay=False, exists=True, readable=True, resolve_path=True), help='file contains bounding boxes or tasks.') -@click.option('--task-index', '-i', +@click.option('--job-index', '-i', type=int, default=None, help='index of task in the tasks.') @click.option('--slurm-job-array/--no-slurm-job-array', @@ -235,17 +235,16 @@ def create_info(tasks,layer_path, channel_num, layer_type, data_type, encoding, 'environment variable to identify task index.') @click.option('--granularity', '-g', type=int, default=1, help='number of tasks to do in one run.') -@operator -def fetch_task_from_file(tasks, file_path, task_index, slurm_job_array, granularity): +@generator +def fetch_task_from_file(file_path: str, job_index: int, slurm_job_array: bool, granularity: int): if(slurm_job_array): - assert os.environ['SLURM_ARRAY_JOB_ID'] == 0 - assert os.environ['SLURM_ARRAY_TASK_ID'] >= 0 - task_index = os.environ['SLURM_ARRAY_TASK_ID'] - assert task_index is not None + job_index = int(os.environ['SLURM_ARRAY_TASK_ID']) + assert job_index is not None bbox_array = np.load(file_path) - task_stop = min(bbox_array.shape[0], task_index + granularity) - for idx in range(task_index, task_stop): + task_start = job_index * granularity + task_stop = min(bbox_array.shape[0], task_start + granularity) + for idx in range(task_start, task_stop): bbox = Bbox.from_list(bbox_array[idx, :]) task = get_initial_task() task['bbox'] = bbox @@ -1230,6 +1229,7 @@ def neuroglancer(tasks, name, voxel_size, port, chunk_names): state['operators'][name](task, selected=chunk_names) yield task + @main.command('quantize') @click.option('--name', type=str, default='quantize', help='name of this operator') @click.option('--input-chunk-name', '-i', type=str, default='chunk', help = 'input chunk name') diff --git a/chunkflow/plugins/mapto01.py b/chunkflow/plugins/mapto01.py index 340af69b..caffaec9 100644 --- a/chunkflow/plugins/mapto01.py +++ b/chunkflow/plugins/mapto01.py @@ -1,6 +1,9 @@ import numpy as np def exec(chunk: np.ndarray): + if np.issubdtype(chunk.dtype, np.uint8): + chunk = chunk.astype(np.float32) + chunk = chunk / 255.0 assert np.issubdtype(chunk.dtype, np.float32) assert chunk.min() >= -1 assert chunk.max() <= 1 diff --git a/requirements.txt b/requirements.txt index 7792c684..c2eb6410 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -python-dateutil==2.8.0 boto3 numpy>=1.16 six>=1.13.0