Skip to content

Commit

Permalink
tech: sonar-cloud changes for formatting issues and "MapAliasses… (#98)
Browse files Browse the repository at this point in the history 
* tech-debt: sonar-cloud changes for formatting issues and "MapAliasses", "ReadCsvValue", "CreateCsvLine" functions.

* Tech-deck: Sonar cloud fixes to remove nulls from execution paths

* Tech-Debt: changed System exception to custom

* Tech-Deck: added unit tests to cover changed lines
  • Loading branch information
@shraines
shraines committed Jan 6, 2023
1 parent c3fee22 commit d3c52ab
Show file tree
Hide file tree
Showing 8 changed files with 188 additions and 112 deletions.
3 changes: 2 additions & 1 deletion src/Capgemini.DataMigration.Core/GenericDataMigrator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ namespace Capgemini.DataMigration.Core
/// <summary>
/// Generic Data Migrator Engine.
/// </summary>
public class GenericDataMigrator<TMigrationEntity, TMigrationEntityWrapper> : IGenericDataMigrator<TMigrationEntity, TMigrationEntityWrapper> where TMigrationEntity : class
public class GenericDataMigrator<TMigrationEntity, TMigrationEntityWrapper> : IGenericDataMigrator<TMigrationEntity, TMigrationEntityWrapper>
where TMigrationEntity : class
where TMigrationEntityWrapper : MigrationEntityWrapper<TMigrationEntity>
{
private readonly IDataStoreWriter<TMigrationEntity, TMigrationEntityWrapper> storeWriter;
Expand Down
2 changes: 2 additions & 0 deletions src/Capgemini.DataMigration.Core/IGenericDataMigrator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ public interface IGenericDataMigrator<TMigrationEntity, TMigrationEntityWrapper>
where TMigrationEntityWrapper : MigrationEntityWrapper<TMigrationEntity>
{
void AddProcessor(IEntityProcessor<TMigrationEntity, TMigrationEntityWrapper> processor);

int GetStartingPassNumber();

void MigrateData();
}
}
4 changes: 2 additions & 2 deletions src/Capgemini.DataMigration.Core/Model/FieldToBeObfuscated.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using Newtonsoft.Json;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;

namespace Capgemini.DataMigration.Core.Model
{
Expand Down
88 changes: 45 additions & 43 deletions src/Capgemini.Xrm.DataMigration.Engine/DataProcessors/MapEntityProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IdentityModel.Metadata;
using System.Linq;
using Capgemini.DataMigration.Core;
using Capgemini.DataMigration.Core.Extensions;
Expand Down Expand Up @@ -108,12 +109,9 @@ public void ImportCompleted()

private static void MapGuidAttributes(EntityWrapper entity, Dictionary<string, Dictionary<Guid, Guid>> mappings, List<KeyValuePair<string, object>> items)
{
foreach (KeyValuePair<string, object> item in items)
foreach (var item in items.Where(item => item.Value is Guid))
{
if (item.Value is Guid)
{
SetEnityAttributeFromMap(entity, mappings, item);
}
SetEnityAttributeFromMap(entity, mappings, item);
}
}

Expand Down Expand Up @@ -337,56 +335,60 @@ private void MapAliasses(EntityWrapper entity, int passNumber)
}

Type fieldType = metCache.GetAttributeDotNetType(entity.LogicalName, originalFieldName);
SetReplacementGuid(fieldType, entity, originalFieldName, replaceGuid, alias);
}

if (fieldType == typeof(Guid))
var entCollections = entity.OriginalEntity.Attributes.Where(a => a.Value is EntityCollection).Select(a => a.Value).ToList();
entCollections.ForEach(item =>
{
EntityCollection entCol = item as EntityCollection;
foreach (var ent in entCol.Entities)
{
if (entity.OriginalEntity.Attributes.ContainsKey(originalFieldName))
{
logger.LogVerbose($"MapAliasses Guid: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, using replaced Guid {replaceGuid}");
entity.OriginalEntity.Attributes[originalFieldName] = replaceGuid;
}
else
{
logger.LogVerbose($"MapAliasses Guid: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, only mapping exists, using replaced Guid {replaceGuid}");
entity.OriginalEntity.Attributes.Add(originalFieldName, replaceGuid);
}
MapAliasses(new EntityWrapper(ent), passNumber);
}
else if (fieldType == typeof(EntityReference))
});
}

private void SetReplacementGuid(Type fieldType, EntityWrapper entity, string originalFieldName, Guid replaceGuid, string alias)
{
if (fieldType == typeof(Guid))
{
if (entity.OriginalEntity.Attributes.ContainsKey(originalFieldName))
{
if (entity.OriginalEntity.Attributes.ContainsKey(originalFieldName))
{
logger.LogVerbose($"MapAliasses EntRef: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, using replaced Guid {replaceGuid}");
var originalEntityReference = (EntityReference)entity.OriginalEntity.Attributes[originalFieldName];
originalEntityReference.Id = replaceGuid;

if (string.IsNullOrEmpty(originalEntityReference.LogicalName))
{
originalEntityReference.LogicalName = metCache.GetLookUpEntityName(entity.LogicalName, originalFieldName);
}
}
else
{
logger.LogVerbose($"MapAliasses EntRef: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, only mapping exists, using replaced Guid {replaceGuid}");
string refEntName = metCache.GetLookUpEntityName(entity.LogicalName, originalFieldName);
EntityReference entRef = new EntityReference(refEntName, replaceGuid);
entity.OriginalEntity.Attributes.Add(originalFieldName, entRef);
}
logger.LogVerbose($"MapAliasses Guid: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, using replaced Guid {replaceGuid}");
entity.OriginalEntity.Attributes[originalFieldName] = replaceGuid;
}
else
{
throw new ConfigurationException($"MapAliasses: Bad Data Migrator Configuration! {alias} Using mapping for unsupported type {fieldType.FullName} !");
logger.LogVerbose($"MapAliasses Guid: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, only mapping exists, using replaced Guid {replaceGuid}");
entity.OriginalEntity.Attributes.Add(originalFieldName, replaceGuid);
}
}

var entCollections = entity.OriginalEntity.Attributes.Where(a => a.Value is EntityCollection).Select(a => a.Value).ToList();
entCollections.ForEach(item =>
else if (fieldType == typeof(EntityReference))
{
EntityCollection entCol = item as EntityCollection;
foreach (var ent in entCol.Entities)
if (entity.OriginalEntity.Attributes.ContainsKey(originalFieldName))
{
MapAliasses(new EntityWrapper(ent), passNumber);
logger.LogVerbose($"MapAliasses EntRef: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, using replaced Guid {replaceGuid}");
var originalEntityReference = (EntityReference)entity.OriginalEntity.Attributes[originalFieldName];
originalEntityReference.Id = replaceGuid;

if (string.IsNullOrEmpty(originalEntityReference.LogicalName))
{
originalEntityReference.LogicalName = metCache.GetLookUpEntityName(entity.LogicalName, originalFieldName);
}
}
});
else
{
logger.LogVerbose($"MapAliasses EntRef: Entity:{entity.LogicalName}:{entity.Id}, Field:{originalFieldName}, only mapping exists, using replaced Guid {replaceGuid}");
string refEntName = metCache.GetLookUpEntityName(entity.LogicalName, originalFieldName);
EntityReference entRef = new EntityReference(refEntName, replaceGuid);
entity.OriginalEntity.Attributes.Add(originalFieldName, entRef);
}
}
else
{
throw new ConfigurationException($"MapAliasses: Bad Data Migrator Configuration! {alias} Using mapping for unsupported type {fieldType.FullName} !");
}
}
}
}
115 changes: 68 additions & 47 deletions src/Capgemini.Xrm.DataMigration.FileStore/DataStore/DataFileStoreWriterCsv.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,28 +114,17 @@ private List<string> GetHeader(CrmExportedDataStore store)
store.ThrowArgumentNullExceptionIfNull(nameof(store));

var ent = store.ExportedEntities.FirstOrDefault();
List<string> header = new List<string>();

if (!ent.IsManyToMany)
if (ent == null)
{
CrmEntity entity = schemaConfig.Entities.FirstOrDefault(p => p.Name == ent.LogicalName);

var fields = entity.CrmFields.Where(p => p.FieldName != entity.PrimaryIdField).Select(p => p.FieldName).ToList();

foreach (var lookup in entity.CrmFields.Where(p => p.FieldName != entity.PrimaryIdField && p.FieldType == "entityreference" && p.FieldName == "ownerid").Select(p => $"{p.FieldName}.LogicalName"))
{
fields.Add(lookup);
}
throw new ConfigurationException($"null value returned from {nameof(store.ExportedEntities)}");
}

AddFieldWithCheck(header, entity.PrimaryIdField);
List<string> header = new List<string>();

if (fields != null)
{
foreach (var field in fields)
{
AddFieldWithCheck(header, field);
}
}
if (!ent.IsManyToMany)
{
AddSingleEntityFields(ent.LogicalName, header);
}
else
{
Expand All @@ -144,6 +133,11 @@ private List<string> GetHeader(CrmExportedDataStore store)

CrmRelationship rel = entity.CrmRelationships.FirstOrDefault(a => a.RelatedEntityName == ent.LogicalName);

if (rel == null)
{
throw new ConfigurationException($"null value returned from {nameof(entity.CrmRelationships)}");
}

AddFieldWithCheck(header, $"{rel.RelationshipName}id");
AddFieldWithCheck(header, entity.PrimaryIdField);
AddFieldWithCheck(header, rel.TargetEntityPrimaryKey);
Expand All @@ -162,6 +156,33 @@ private List<string> GetHeader(CrmExportedDataStore store)
return header;
}

private void AddSingleEntityFields(string entLogicalName, List<string> header)
{
CrmEntity entity = schemaConfig.Entities.FirstOrDefault(p => p.Name == entLogicalName);

if (entity == null)
{
throw new ConfigurationException($"null value returned from {nameof(schemaConfig.Entities)}");
}

var fields = entity.CrmFields.Where(p => p.FieldName != entity.PrimaryIdField).Select(p => p.FieldName).ToList();

foreach (var lookup in entity.CrmFields.Where(p => p.FieldName != entity.PrimaryIdField && p.FieldType == "entityreference" && p.FieldName == "ownerid").Select(p => $"{p.FieldName}.LogicalName"))
{
fields.Add(lookup);
}

AddFieldWithCheck(header, entity.PrimaryIdField);

if (fields != null)
{
foreach (var field in fields)
{
AddFieldWithCheck(header, field);
}
}
}

private void AddFieldWithCheck(List<string> header, string fieldName)
{
if (!excludedFields.Contains(fieldName))
Expand All @@ -177,44 +198,44 @@ private string CreateCsvHeader(List<string> header)

private string CreateCsvLine(CrmEntityStore record, List<string> header)
{
if (record != null)
if (record == null)
{
string[] items = new string[header.Count];
items[0] = record.Id.ToString();
foreach (var item in record.Attributes)
{
int position = header.IndexOf(item.AttributeName);
return null;
}

if (position >= 0)
string[] items = new string[header.Count];
items[0] = record.Id.ToString();
foreach (var item in record.Attributes)
{
int position = header.IndexOf(item.AttributeName);

if (position >= 0)
{
object attrValue = EntityConverterHelper.GetAttributeValueForCsv(item);
if (item.AttributeType == "System.String")
{
object attrValue = EntityConverterHelper.GetAttributeValueForCsv(item);
if (item.AttributeType == "System.String")
{
string atVal = attrValue?.ToString().Replace("\"", "\"\"");
items[position] = $"\"{atVal}\"";
}
else if (item.AttributeType == "Microsoft.Xrm.Sdk.EntityReference" && item.AttributeName == "ownerid")
{
var entityref = (EntityReference)item.AttributeValue;
int logicalNamePosition = header.IndexOf($"{item.AttributeName}.LogicalName");
if (logicalNamePosition >= 0)
{
items[logicalNamePosition] = $"\"{entityref.LogicalName}\"";
}

items[position] = attrValue?.ToString();
}
else
string atVal = attrValue?.ToString().Replace("\"", "\"\"");
items[position] = $"\"{atVal}\"";
}
else if (item.AttributeType == "Microsoft.Xrm.Sdk.EntityReference" && item.AttributeName == "ownerid")
{
var entityref = (EntityReference)item.AttributeValue;
int logicalNamePosition = header.IndexOf($"{item.AttributeName}.LogicalName");
if (logicalNamePosition >= 0)
{
items[position] = attrValue?.ToString();
items[logicalNamePosition] = $"\"{entityref.LogicalName}\"";
}

items[position] = attrValue?.ToString();
}
else
{
items[position] = attrValue?.ToString();
}
}

return string.Join(delimiter, items);
}

return null;
return string.Join(delimiter, items);
}

private string GetFileNameForBatchNo(int batchNo, string entName)
Expand Down
40 changes: 25 additions & 15 deletions src/Capgemini.Xrm.DataMigration.FileStore/Helpers/ReadCsvHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using Capgemini.DataMigration.Exceptions;
using Capgemini.Xrm.DataMigration.Config;
using Capgemini.Xrm.DataMigration.DataStore;
using Capgemini.Xrm.DataMigration.Model;
using CsvHelper;
using Microsoft.Xrm.Sdk;

Expand All @@ -26,6 +27,12 @@ public List<EntityWrapper> ReadFromFile(string fileName, string entityName)
{
List<EntityWrapper> store = new List<EntityWrapper>();
string headerLine = File.ReadLines(fileName).FirstOrDefault();

if (headerLine == null)
{
throw new ConfigurationException($"null data retruned from file: {fileName}");
}

List<string> header = headerLine.Split(',').ToList();

using (TextReader tr = File.OpenText(fileName))
Expand Down Expand Up @@ -82,23 +89,11 @@ private static void ReadAliassedValue(CsvReader reader, Entity entity, int idx,

private void ReadCsvValue(string entityName, CsvReader reader, Entity entity, int idx, string attrName, List<string> header)
{
string fieldName = attrName;

var field = schemaConfig.Entities.Single(p => p.Name == entityName).CrmFields.FirstOrDefault(p => p.FieldName == fieldName);
var field = schemaConfig.Entities.Single(p => p.Name == entityName).CrmFields.FirstOrDefault(p => p.FieldName == attrName);

if (fieldName == "ownerid")
if (attrName == "ownerid")
{
string lookupType;
int colIndex = header.FindIndex(x => x == "ownerid.LogicalName");

if (colIndex == -1 || !reader.TryGetField(colIndex, out lookupType))
{
logger.LogWarning("CSV file does not contain column ownerid.LogicalName! OwnerId will be mapped to systemuser. If you wanted granular mapping of OwnerId, please regenerate the CSV.");
}
else
{
field.LookupType = lookupType;
}
GetOwnerIdLogicalName(header, reader, field);
}

if (field != null)
Expand All @@ -121,5 +116,20 @@ private void ReadCsvValue(string entityName, CsvReader reader, Entity entity, in
}
}
}

private void GetOwnerIdLogicalName(List<string> header, CsvReader reader, CrmField field)
{
string lookupType;
int colIndex = header.FindIndex(x => x == "ownerid.LogicalName");

if (colIndex == -1 || !reader.TryGetField(colIndex, out lookupType))
{
logger.LogWarning("CSV file does not contain column ownerid.LogicalName! OwnerId will be mapped to systemuser. If you wanted granular mapping of OwnerId, please regenerate the CSV.");
}
else
{
field.LookupType = lookupType;
}
}
}
}
Loading

0 comments on commit d3c52ab

Please sign in to comment.