Skip to content

Commit 12c52cd

Browse files
mslabkomslabko
authored
MDEE-936 poc to support CCDM to MDEE (#460)
* MDEE-936: extend schema - allow to disable specific field for feed - filter by already deleted entities when select IDs that should be marked for deletion - pass metadata to IndexStateProvider for extensibility purpose - filter out feed with wrong type
1 parent 4fad67d commit 12c52cd

10 files changed

+73
-31
lines changed

‎DataExporter/Config/Converter.php

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,20 @@ public function convert($source)
6262
];
6363
$idField = null;
6464
foreach ($queryData['field'] as $fieldData) {
65+
$disabled = isset($fieldData['disabled']) && $fieldData['disabled'] == 'true';
66+
if ($disabled) {
67+
continue;
68+
}
6569
$field = [
6670
'name' => $fieldData['name'],
6771
'type' => $fieldData['type'],
68-
'provider' => isset($fieldData['provider']) ? $fieldData['provider'] : null,
69-
'repeated' => (isset($fieldData['repeated']) && $fieldData['repeated'] == "true") ? true : false
72+
'provider' => isset($fieldData['provider']) && $fieldData['provider'] !== 'null' ? $fieldData['provider'] : null,
73+
'repeated' => isset($fieldData['repeated']) && $fieldData['repeated'] == 'true'
7074
];
7175
if ($fieldData['type'] == 'ID') {
7276
$idField = $fieldData['name'];
7377
}
74-
if (isset($fieldData['provider'])) {
78+
if (isset($field['provider'])) {
7579
if (isset($fieldData['using'])) {
7680
foreach ($fieldData['using'] as $usingField) {
7781
$field['using'][$usingField['field']] = $usingField;

‎DataExporter/Model/FeedMetadataPool.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class FeedMetadataPool
3434
public function __construct(
3535
array $classMap = []
3636
) {
37-
$this->classMap = $classMap;
37+
$this->classMap = array_filter($classMap, fn($feed) => $feed instanceof FeedIndexMetadata);
3838
}
3939

4040
/**

‎DataExporter/Model/Indexer/DataSerializer.php

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,7 @@ private function serializeForImmediateExport(
116116
$output = [];
117117
$status = $exportStatus->getStatus();
118118
$exportFailedItems = $exportStatus->getFailedItems();
119-
$feedItemFields = array_values($metadata->getFeedItemIdentifiers());
120-
$feedItemFieldsToPersist = array_merge(
121-
$metadata->getMinimalPayloadFieldsList(),
122-
// required to build feed's table primary key if entity was deleted
123-
array_combine($feedItemFields, $feedItemFields),
124-
);
119+
$feedItemFieldsToPersist = $metadata->getMinimalPayloadFieldsList();
125120
$rowModifiedAt = (new \DateTime())->format($metadata->getDbDateTimeFormat());
126121

127122
$itemN = -1;
@@ -132,7 +127,7 @@ private function serializeForImmediateExport(
132127
foreach ($this->unserializeKeys as $unserializeKey) {
133128
$feedData[$unserializeKey] = $this->serializer->unserialize($feedData[$unserializeKey]);
134129
}
135-
$sourceEntityId = $feedData[$metadata->getFeedIdentity()] ?? null;
130+
$sourceEntityId = $row[FeedIndexMetadata::FEED_TABLE_FIELD_SOURCE_ENTITY_ID] ?? null;
136131
if ($sourceEntityId === null) {
137132
$this->logger->warning(
138133
'Source entity id is null. Check your data. field: %s, Feed data: %s',
@@ -148,12 +143,12 @@ private function serializeForImmediateExport(
148143
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_STATUS] = $status->getValue();
149144
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_MODIFIED_AT] = $rowModifiedAt;
150145
if (!empty($exportFailedItems)) {
151-
$failedFeedItem = $exportFailedItems[$itemN]['message'] ?? null;
152-
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_ERRORS] = $failedFeedItem ?? '';
146+
$failedFeedItem = $exportFailedItems[$itemN] ?? null;
153147
// if _specific_ item failed mark only that item as failed, otherwise set status successful
154148
if ($failedFeedItem !== null) {
155-
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_STATUS]
156-
= ExportStatusCodeProvider::FAILED_ITEM_ERROR;
149+
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_ERRORS] = $failedFeedItem['message'] ?? '';
150+
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_STATUS] = $failedFeedItem['status']
151+
?? ExportStatusCodeProvider::FAILED_ITEM_ERROR;
157152
}
158153
} elseif (!$status->isSuccess()) {
159154
$outputRow[FeedIndexMetadata::FEED_TABLE_FIELD_ERRORS] = $exportStatus->getReasonPhrase();

‎DataExporter/Model/Indexer/FeedIndexMetadata.php

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -438,11 +438,7 @@ public function isPersistExportedFeed(): bool
438438
*/
439439
public function getMinimalPayloadFieldsList(): array
440440
{
441-
return array_merge(
442-
$this->minimalPayload,
443-
// Feed identity is a required field to build feed item
444-
[$this->getFeedIdentity() => $this->getFeedIdentity()]
445-
);
441+
return array_filter($this->minimalPayload);
446442
}
447443

448444
/**

‎DataExporter/Model/Indexer/FeedIndexProcessorCreateUpdate.php

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ public function partialReindex(
114114
): void {
115115
$isPartialReindex = $indexState === null;
116116
if ($isPartialReindex) {
117-
$indexState = $this->indexStateProviderFactory->create(['batchSize' => $metadata->getBatchSize()]);
117+
$indexState = $this->indexStateProviderFactory->create(['metadata' => $metadata]);
118118
}
119119

120120
$feedIdentity = $metadata->getFeedIdentity();
@@ -178,7 +178,7 @@ public function fullReindex(
178178
$userFunctions = [];
179179
for ($threadNumber = 1; $threadNumber <= $threadCount; $threadNumber++) {
180180
$userFunctions[] = function () use ($batchIterator, $metadata, $serializer, $idsProvider) {
181-
$indexState = $this->indexStateProviderFactory->create(['batchSize' => $metadata->getBatchSize()]);
181+
$indexState = $this->indexStateProviderFactory->create(['metadata' => $metadata]);
182182
try {
183183
foreach ($batchIterator as $ids) {
184184
$this->partialReindex($metadata, $serializer, $idsProvider, $ids, null, $indexState);
@@ -285,6 +285,7 @@ private function filterFeedItems(
285285
['f' => $this->resourceConnection->getTableName($metadata->getFeedTableName())],
286286
[
287287
FeedIndexMetadata::FEED_TABLE_FIELD_PK,
288+
FeedIndexMetadata::FEED_TABLE_FIELD_SOURCE_ENTITY_ID,
288289
FeedIndexMetadata::FEED_TABLE_FIELD_FEED_ID,
289290
FeedIndexMetadata::FEED_TABLE_FIELD_FEED_HASH,
290291
FeedIndexMetadata::FEED_TABLE_FIELD_STATUS,
@@ -306,6 +307,8 @@ private function filterFeedItems(
306307
)) {
307308
$feedItems[$identifier][FeedIndexMetadata::FEED_TABLE_FIELD_PK]
308309
= $row[FeedIndexMetadata::FEED_TABLE_FIELD_PK];
310+
$feedItems[$identifier][FeedIndexMetadata::FEED_TABLE_FIELD_SOURCE_ENTITY_ID]
311+
= $row[FeedIndexMetadata::FEED_TABLE_FIELD_SOURCE_ENTITY_ID];
309312
$feedItems[$identifier][FeedIndexMetadata::FEED_TABLE_FIELD_STATUS]
310313
= $row[FeedIndexMetadata::FEED_TABLE_FIELD_STATUS];
311314
$updates[] = $feedItems[$identifier];
@@ -391,6 +394,8 @@ private function addHashesAndModifiedAt(array $data, FeedIndexMetadata $metadata
391394
}
392395
$hash = $this->hashBuilder->buildHash($row, $metadata);
393396
$data[$identifier] = [
397+
// source entity id required only to persist data to table, but we still may send feed item
398+
FeedIndexMetadata::FEED_TABLE_FIELD_SOURCE_ENTITY_ID => $row[$metadata->getFeedIdentity()] ?? null,
394399
FeedIndexMetadata::FEED_TABLE_FIELD_FEED_ID => $identifier,
395400
FeedIndexMetadata::FEED_TABLE_FIELD_FEED_HASH => $hash,
396401
FeedIndexMetadata::FEED_TABLE_FIELD_FEED_DATA => $row,

‎DataExporter/Model/Indexer/IndexStateProvider.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@ class IndexStateProvider
2222
public const UPDATE_OPERATION = 2;
2323

2424
/**
25-
* @param int $batchSize
25+
* @param FeedIndexMetadata $metadata
2626
*/
27-
public function __construct(int $batchSize)
27+
public function __construct(FeedIndexMetadata $metadata)
2828
{
29-
$this->batchSize = $batchSize;
29+
$this->batchSize = $metadata->getBatchSize();
3030
}
3131

3232
/**

‎DataExporter/Model/Query/DeletedEntitiesByModifiedAtQuery.php

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ public function getQuery(array $ids, FeedIndexMetadata $metadata, string $recent
4848
['f' => $this->resourceConnection->getTableName($metadata->getFeedTableName())]
4949
)
5050
->where(\sprintf('f.%s IN (?)', $metadata->getFeedTableField()), $ids)
51-
->where('f.modified_at < ?', $recentTimeStamp);
51+
->where('f.modified_at < ?', $recentTimeStamp)
52+
// return only un-deleted items to prevent extra processing
53+
->where('f.is_deleted = ?', 0);
5254
}
5355
}

‎DataExporter/Status/ExportStatusCode.php

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@
1616

1717
namespace Magento\DataExporter\Status;
1818

19+
/**
20+
* Original status (HTTP status or application) code is mapped to one of the following Export Status codes:
21+
* - HTTP status code 200 (success)
22+
* - HTTP status code 400 (user error)
23+
* - Application status "skip submission ("-1"), @see ExportStatusCodeProvider::FEED_SUBMIT_SKIPPED
24+
* - Application status "retryable ("2"), @see ExportStatusCodeProvider::RETRYABLE
25+
*
26+
* Note, any HTTP status code different from 200 or 400 will be considered as retryable
27+
*/
1928
class ExportStatusCode
2029
{
2130
/**
@@ -24,18 +33,26 @@ class ExportStatusCode
2433
private const SUCCESS = 200;
2534

2635
/**
27-
* Value of current status code
36+
* Export operation status code
2837
* @var int
2938
*/
3039
private int $statusCode;
3140

41+
/**
42+
* Keeps original status code (HTTP status or application status code)
43+
*
44+
* @var int
45+
*/
46+
private int $originalStatusCode;
47+
3248
/**
3349
* @param int $statusCode
3450
*/
3551
public function __construct(
3652
int $statusCode
3753
) {
38-
$this->statusCode = $statusCode;
54+
$this->originalStatusCode = $statusCode;
55+
$this->statusCode = $this->mapStatus($statusCode);
3956
}
4057

4158
/**
@@ -49,14 +66,14 @@ public function isSuccess(): bool
4966
}
5067

5168
/**
52-
* Return true regardless of the response status code
69+
* Return true if data was sent outside AC regardless of the response status code
5370
*
5471
* @return bool
5572
*/
5673
public function isSent(): bool
5774
{
5875
return !in_array(
59-
$this->statusCode,
76+
$this->originalStatusCode,
6077
[
6178
ExportStatusCodeProvider::APPLICATION_ERROR,
6279
ExportStatusCodeProvider::FEED_SUBMIT_SKIPPED
@@ -74,4 +91,20 @@ public function getValue(): int
7491
{
7592
return $this->statusCode;
7693
}
94+
95+
/**
96+
* @param int $statusCode
97+
* @return int
98+
*/
99+
private function mapStatus(int $statusCode): int
100+
{
101+
return in_array($statusCode, ExportStatusCodeProvider::NON_RETRYABLE_HTTP_STATUS_CODE, true)
102+
|| in_array(
103+
$statusCode,
104+
[ExportStatusCodeProvider::RETRYABLE, ExportStatusCodeProvider::FEED_SUBMIT_SKIPPED],
105+
true
106+
)
107+
? $statusCode
108+
: ExportStatusCodeProvider::RETRYABLE;
109+
}
77110
}

‎DataExporter/Status/ExportStatusCodeProvider.php

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
class ExportStatusCodeProvider
2020
{
2121
/**
22-
* List of non-retryable HTTP codes
22+
* List of non-retryable HTTP status codes.
23+
* Until item hash is changed, feed item will not be sent again on sync attempt
2324
*/
2425
public const NON_RETRYABLE_HTTP_STATUS_CODE = [200, 400];
2526

@@ -40,4 +41,9 @@ class ExportStatusCodeProvider
4041
* in case if FEED_SUBMIT_SKIPPED response status was returned - the DB feed saving process will be skipped
4142
*/
4243
public const FEED_SUBMIT_SKIPPED = -1;
44+
45+
/**
46+
* Custom code to identify retryable operation (for entire request or for specific item)
47+
*/
48+
public const RETRYABLE = 2;
4349
}

‎DataExporter/etc/et_schema.xsd

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
<xs:attribute name="provider" type="xs:string" />
5959
<xs:attribute name="type" type="xs:string" />
6060
<xs:attribute name="repeated" type="xs:boolean" />
61+
<xs:attribute name="disabled" type="xs:boolean" />
6162
</xs:complexType>
6263
<xs:complexType name="Using">
6364
<xs:attribute name="field" type="xs:string" use="required" />

0 commit comments

Comments
 (0)