-
-
Notifications
You must be signed in to change notification settings - Fork 3.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fixes part of #18442: Remove deprecated edit state content suggestion model #20079
Changes from 8 commits
fbfde4b
ee7ed03
451ee1e
a82ee41
c509608
59659a5
cf130cb
e2e9297
4335287
d6b6850
20fb747
926dc2b
fdaf85a
d80039d
ab4829f
d52ea67
5ff1b63
bb11d07
91b9826
c7c63ea
cbbd796
ccf71de
6e44674
a7ae87f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,123 @@ | ||
# coding: utf-8 | ||
# | ||
# Copyright 2024 The Oppia Authors. All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS-IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
"""Deletion Jobs for edit state content suggestion models""" | ||
|
||
from __future__ import annotations | ||
|
||
from core import feconf | ||
from core.jobs import base_jobs | ||
from core.jobs.io import ndb_io | ||
from core.jobs.transforms import job_result_transforms | ||
from core.jobs.types import job_run_result | ||
from core.platform import models | ||
|
||
import apache_beam as beam | ||
from typing import Tuple | ||
|
||
MYPY = False | ||
if MYPY: # pragma: no cover | ||
from mypy_imports import suggestion_models | ||
|
||
(suggestion_models, ) = models.Registry.import_models([ | ||
models.Names.SUGGESTION]) | ||
|
||
|
||
# TODO(#15613): Here we use MyPy ignore because the incomplete typing of | ||
# apache_beam library and absences of stubs in Typeshed, forces MyPy to | ||
# assume that PTransform class is of type Any. Thus to avoid MyPy's error | ||
# (Class cannot subclass 'PTransform' (has type 'Any')), we added an | ||
# ignore here. | ||
class GetDeprecatedSuggestionEditStateContentModels(beam.PTransform): # type: ignore[misc] | ||
"""Transform that gets all edit state content suggestion models.""" | ||
|
||
def expand( | ||
self, pipeline: beam.Pipeline | ||
) -> Tuple[ | ||
beam.PCollection[suggestion_models.GeneralSuggestionModel], | ||
beam.PCollection[job_run_result.JobRunResult] | ||
]: | ||
suggestion_edit_state_content_model_to_delete = ( | ||
pipeline | ||
| 'Get all general suggestion models' >> ndb_io.GetModels( | ||
suggestion_models.GeneralSuggestionModel.get_all()) | ||
| 'Filter edit state content suggestion' >> ( | ||
beam.Filter( | ||
lambda model: (( | ||
model.suggestion_type == ( | ||
feconf.SUGGESTION_TYPE_EDIT_STATE_CONTENT)) and ( | ||
model.target_type == ( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do you actually need this check? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here, first we are fetching all the general suggestion models the filtering needed models using |
||
feconf.ENTITY_TYPE_EXPLORATION))) | ||
)) | ||
) | ||
|
||
suggestion_edit_state_content_model_to_delete_count = ( | ||
suggestion_edit_state_content_model_to_delete | ||
| 'Count edit state content suggestion to be deleted' >> ( | ||
job_result_transforms.CountObjectsToJobRunResult( | ||
'EDIT STATE CONTENT SUGGESTION')) | ||
) | ||
|
||
return ( | ||
suggestion_edit_state_content_model_to_delete, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Deindent this and the next line by 4 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
suggestion_edit_state_content_model_to_delete_count, | ||
) | ||
|
||
|
||
class DeleteDeprecatedSuggestionEditStateContentModelsJob(base_jobs.JobBase): | ||
"""Job that deletes edit state content suggestion models as these are | ||
deprecated. | ||
""" | ||
|
||
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: | ||
|
||
(suggestion_edit_state_content_model_to_delete, ( | ||
suggestion_edit_state_content_model_to_delete_result)) = ( | ||
self.pipeline | ||
| 'Get edit state content suggestion models' >> ( | ||
GetDeprecatedSuggestionEditStateContentModels()) | ||
) | ||
|
||
unused_models_deletion = ( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unused_models_deletion_result Though, if it's unused, then why bother to declare it in the first place? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We need to assign expression There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, can you rename to unused_models_deletion_result then (as stated in the earlier comment)? |
||
( | ||
suggestion_edit_state_content_model_to_delete | ||
) | ||
| 'Extract keys' >> beam.Map(lambda model: model.key) | ||
| 'Delete models' >> ndb_io.DeleteModels() | ||
) | ||
|
||
return ( | ||
( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think you need this parens and the indentation below? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
suggestion_edit_state_content_model_to_delete_result, | ||
) | ||
| 'Merge results' >> beam.Flatten() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just wondering, why do we not report these the same way as in the audit job? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Totally agree, we actually don't need to use beam.Flatten() here. |
||
) | ||
|
||
|
||
class AuditDeprecatedSuggestionEditStateContentModelsDeletionJob( | ||
base_jobs.JobBase): | ||
"""Job that audit edit state content suggestion.""" | ||
|
||
def run(self) -> beam.PCollection[job_run_result.JobRunResult]: | ||
|
||
job_run_results = ( | ||
self.pipeline | ||
| 'Perform fetching and deletion of edit state content' + | ||
' suggestion results' >> ( | ||
GetDeprecatedSuggestionEditStateContentModels()) | ||
)[1] | ||
|
||
return job_run_results |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Deletion jobs ...
End the docstring with a period.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
done