diff --git a/config.yaml.sample b/config.yaml.sample index 29c4c1a5..e0ead894 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -388,3 +388,7 @@ batch: retry-attempts: 5 # check store if metatile already exists, and if so skip processing check-metatile-exists: true + # optional override memory reserved for container when submiting job + # NOTE: in megabytes + # https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html + memory: diff --git a/tilequeue/command.py b/tilequeue/command.py index 0e05baa0..aee42f8a 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -1895,6 +1895,7 @@ def tilequeue_batch_enqueue(cfg, peripherals): assert job_queue, 'Missing batch job-queue config' retry_attempts = batch_yaml.get('retry-attempts') + memory = batch_yaml.get('memory') dim = 2 ** queue_zoom z = queue_zoom @@ -1911,22 +1912,23 @@ def tilequeue_batch_enqueue(cfg, peripherals): '--config', '/etc/tilequeue/config.yaml', '--tile', coord_str, ] + container_overrides = dict(command=job_cmd) job_opts = dict( jobDefinition=job_def, jobQueue=job_queue, jobName=job_name, - containerOverrides={ - 'command': job_cmd, - }, + containerOverrides=container_overrides, ) if retry_attempts is not None: job_opts['retryStrategy'] = dict(attempts=retry_attempts) + if memory: + container_overrides['memory'] = memory resp = client.submit_job(**job_opts) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200, \ 'Failed to submit job: %s' % 'JobName' i += 1 - if i % 10000 == 0: - print i + if i % 1000 == 0: + print '%d jobs submitted' % i logger.info('Batch enqueue ... done')