Multiprocessing embedding #241
Conversation
Codecov Report
@@ Coverage Diff @@
## master #241 +/- ##
==========================================
+ Coverage 86.11% 86.19% +0.07%
==========================================
Files 75 75
Lines 4343 4367 +24
==========================================
+ Hits 3740 3764 +24
Misses 603 603
Continue to review full report at Codecov.
|
| partial_train = partial(self._train_one_batch, batch=batch, *args, **kwargs) | ||
| self._models = pool.map(partial_train, self._models) | ||
|
|
||
| elif set(self.model_type) == set(['doc2vec']): |
There was a problem hiding this comment.
self.model_type should already be a set from the constructor right?
skills_ml/utils/__init__.py
Outdated
| if isinstance(x, datetime.datetime) or isinstance(x, datetime.date): | ||
| return x.isoformat() | ||
| if isinstance(x, np.ndarray): | ||
| return len(x) |
There was a problem hiding this comment.
So any arrays of the same length will hash the same way?
| self._model.model_name = self.model_name | ||
| logging.info(f"{', '.join([m.model_name for m in self._models])} are trained in {str(timedelta(seconds=time()-tic))}") | ||
|
|
||
| def _model_hash(self, model, training_time, corpus_metadata): |
There was a problem hiding this comment.
This could probably just be a standalone function because there are no references to self
There was a problem hiding this comment.
Or make it so the caller doesn't have to pass in self.training_time or self.corpus_metadata
| reiter_corpus_gen = Reiterable(corpus_gen) | ||
| self._model.build_vocab(reiter_corpus_gen) | ||
| self._model.train(reiter_corpus_gen, total_examples=self._model.corpus_count, epochs=self._model.iter, *args, **kwargs) | ||
| tic = time() |
There was a problem hiding this comment.
What does tic mean? I would just call this start_time or train_start_time
|
|
||
| def save_model(self, storage=None): | ||
| if storage is None: | ||
| if self.model_storage is None: | ||
| raise AttributeError(f"'self.model_storage' should not be None if you want to save the model") | ||
| ms = self.model_storage | ||
| ms.save_model(self._model, self.model_name) | ||
| for model in self._models: |
There was a problem hiding this comment.
Unless I'm missing something, we cold reduce the branching by dong something like:
if storage:
ms = ModelStorage(storage)
else:
ms = self.model_storage
for model in self._models:
ms.save_model
| self._model_hash(model, self.training_time, self.corpus_metadata)] | ||
| ) + '.model' | ||
|
|
||
| if self.model_type <= set(['word2vec', 'fasttext']): |
There was a problem hiding this comment.
This method is kind of long, maybe splitting this big middle section into _train_batches and _train_full_corpus would work
EmbeddingTrainercorpus_generatorshould be provided in thetrainmethod arguments.BatchGeneratorprovide the same functionality ofbatches_generatorbut can be serialized.