Skip to content

Commit

Permalink
fix: make bytewax settings unexposed
Browse files Browse the repository at this point in the history
Signed-off-by: Hai Nguyen <quanghai.ng1512@gmail.com>
  • Loading branch information
sudohainguyen authored and achals committed Nov 14, 2023
1 parent 5983f40 commit ae1bb8b
Showing 1 changed file with 2 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,6 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel):
mini_batch_size: int = 1000
""" (optional) Number of rows to process per write operation (default 1000)"""

bytewax_replicas: int = 5
""" (optional) Number of process to spawn in each pods to handle a file in parallel"""

bytewax_worker_per_process: int = 1
""" (optional) Number of threads as worker per bytewax process"""

active_deadline_seconds: int = 86400
""" (optional) Maximum amount of time a materialization job is allowed to run"""

Expand Down Expand Up @@ -353,7 +347,7 @@ def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0):
{"name": "BYTEWAX_WORKDIR", "value": "/bytewax"},
{
"name": "BYTEWAX_WORKERS_PER_PROCESS",
"value": f"{self.batch_engine_config.bytewax_worker_per_process}",
"value": "1",
},
{
"name": "BYTEWAX_POD_NAME",
Expand All @@ -366,7 +360,7 @@ def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0):
},
{
"name": "BYTEWAX_REPLICAS",
"value": f"{self.batch_engine_config.bytewax_replicas}",
"value": "1",
},
{
"name": "BYTEWAX_KEEP_CONTAINER_ALIVE",
Expand Down

0 comments on commit ae1bb8b

Please sign in to comment.