Skip to content

Commit

Permalink
#87 refactor verification process in easy upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidBlaa
DavidBlaa committed Jul 17, 2018
1 parent 0237eb1 commit 9c5d8a1
Show file tree
Hide file tree
Showing 20 changed files with 1,380 additions and 1,234 deletions.
28 changes: 13 additions & 15 deletions Components/UI/BExIS.UI/Helpers/JsonTableGenerator.cs
View file
@@ -1,17 +1,15 @@
using DocumentFormat.OpenXml;
using BExIS.IO.DataType.DisplayPattern;
using BExIS.Utils.Models;
using DocumentFormat.OpenXml;
using DocumentFormat.OpenXml.Packaging;
using System.IO;
using System.Linq;
using System.Web.UI.WebControls;
using DocumentFormat.OpenXml.Spreadsheet;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using DocumentFormat.OpenXml.Spreadsheet;
using BExIS.Utils.Models;
using Newtonsoft.Json;
using BExIS.IO.DataType.DisplayPattern;
using System.Diagnostics;

namespace BExIS.UI.Helpers
{
Expand Down Expand Up @@ -39,13 +37,13 @@ public string GenerateJsonTable(SheetFormat sheetFormat, String worksheetUri)
WorkbookPart workbookPart = spreadsheetDocument.WorkbookPart;
_sharedStrings = workbookPart.SharedStringTablePart.SharedStringTable.Elements<SharedStringItem>().ToArray();
_stylesheet = workbookPart.WorkbookStylesPart.Stylesheet;

WorksheetPart worksheetPart = null;
foreach (Sheet worksheet in workbookPart.Workbook.Descendants<Sheet>())
{
//Get the current worksheetpart and see if it is the correct one
WorksheetPart tmp = (WorksheetPart)workbookPart.GetPartById(worksheet.Id);
if(tmp.Uri.ToString() == worksheetUri)
if (tmp.Uri.ToString() == worksheetUri)
{
//Found the correct WorksheetPart
worksheetPart = tmp;
Expand All @@ -68,7 +66,7 @@ public string GenerateJsonTable(SheetFormat sheetFormat, String worksheetUri)

//Since this library will ignore empty rows, check if we skipped some and add empty rows if necessary
//This will still ignore empty rows at the end of the file but those wouldn't have any influence on the indices of data & header anyway
while(row.RowIndex > expectedRowIndex)
while (row.RowIndex > expectedRowIndex)
{
List<String> dummyRow = new List<string>();
dummyRow.Add("");
Expand Down Expand Up @@ -118,7 +116,7 @@ public string GenerateJsonTable(SheetFormat sheetFormat, String worksheetUri)
//If cell contains boolean (doesn't always work for files saved with libre office)
else if (c.DataType != null && c.DataType.HasValue && c.DataType.Value == CellValues.Boolean)
{
if(c.InnerText == "1")
if (c.InnerText == "1")
{
value = "true";
}
Expand All @@ -136,7 +134,7 @@ public string GenerateJsonTable(SheetFormat sheetFormat, String worksheetUri)
{
uint numberFormatId = cellFormat.NumberFormatId.Value;

NumberingFormat numberFormat = _stylesheet.NumberingFormats.FirstOrDefault(numFormat => ((NumberingFormat)numFormat).NumberFormatId.Value == numberFormatId) as NumberingFormat;
NumberingFormat numberFormat = _stylesheet.NumberingFormats?.FirstOrDefault(numFormat => ((NumberingFormat)numFormat).NumberFormatId.Value == numberFormatId) as NumberingFormat;

//
if (numberFormat != null)
Expand All @@ -154,7 +152,7 @@ public string GenerateJsonTable(SheetFormat sheetFormat, String worksheetUri)
DateTime dateTime = DateTime.FromOADate(double.Parse(c.CellValue.Text, CultureInfo.InvariantCulture));
//value = dateTime.ToString(new CultureInfo("en-us"));
//get c# display pattern

DataTypeDisplayPattern dataTypeDisplayPattern = DataTypeDisplayPattern.GetByExcelPattern(formatCode);
value = dataTypeDisplayPattern != null ? dateTime.ToString(dataTypeDisplayPattern.StringPattern) : dateTime.ToString(new CultureInfo("en-us"));

Expand Down
15 changes: 8 additions & 7 deletions Console/BExIS.Web.Shell/Areas/DCM/BExIS.Modules.Dcm.UI.csproj
View file
Expand Up @@ -15,7 +15,7 @@
<RootNamespace>BExIS.Modules.Dcm.UI</RootNamespace>
<AssemblyName>BExIS.Modules.Dcm.UI</AssemblyName>
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<MvcBuildViews>false</MvcBuildViews>
<MvcBuildViews>true</MvcBuildViews>
<UseIISExpress>true</UseIISExpress>
<IISExpressSSLPort />
<IISExpressAnonymousAuthentication />
Expand Down Expand Up @@ -558,6 +558,7 @@
<Content Include="Views\Form\_metadataAttributeViewOffline.cshtml" />
<Content Include="Views\Attachments\Index.cshtml" />
<Content Include="Views\Attachments\_datasetAttachements.cshtml" />
<Content Include="Views\EasyUploadVerification\Row.cshtml" />
<None Include="Web.Debug.config">
<DependentUpon>Web.config</DependentUpon>
</None>
Expand Down Expand Up @@ -700,13 +701,13 @@
<Error Condition="Exists('..\..\..\..\packages\PostSharp.4.3.19\tools\PostSharp.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://www.postsharp.net/links/nuget-restore." />
</Target>
<PropertyGroup>
<PostBuildEvent>mkdir "$(SolutionDir)Console\Workspace\Modules\DCM"
<PostBuildEvent>mkdir "$(SolutionDir)Console\Workspace\Modules\DCM"
C:\Windows\System32\xcopy "$(ProjectDir)Dcm.Settings.xml" "$(SolutionDir)Console\Workspace\Modules\DCM" /C /Y /I</PostBuildEvent>
</PropertyGroup>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target> -->
</Project>
13 changes: 6 additions & 7 deletions Console/BExIS.Web.Shell/Areas/DCM/Controllers/EasyUploadSelectAreasController.cs
View file
Expand Up @@ -3,15 +3,13 @@
using BExIS.IO.Transform.Validation.Exceptions;
using BExIS.Modules.Dcm.UI.Models;
using BExIS.UI.Helpers;
using BExIS.Utils.Helpers;
using BExIS.Utils.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Web.Mvc;
using System.Web.Routing;
using System.Web.Script.Serialization;
using Vaiona.Logging;

namespace BExIS.Modules.Dcm.UI.Controllers
Expand All @@ -34,14 +32,14 @@ public ActionResult SelectAreas(int index)
TaskManager.RemoveExecutedStep(TaskManager.Current());
}


//Use the given file and the given sheet format to create a json-table
string filePath = TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString();
FileStream fis = null;
string jsonTable = "[]";

SelectAreasModel model = new SelectAreasModel();

try
{
//FileStream for the users file
Expand Down Expand Up @@ -70,7 +68,7 @@ public ActionResult SelectAreas(int index)

//Save the worksheet uris to the model
model.SheetUriDictionary = EUEReader.GetWorksheetUris();

if (!String.IsNullOrEmpty(jsonTable))
{
TaskManager.AddToBus(EasyUploadTaskManager.SHEET_JSON_DATA, jsonTable);
Expand All @@ -90,7 +88,7 @@ public ActionResult SelectAreas(int index)
fis.Close();
}
}

// Check if the areas have already been selected, if yes, use them (Important when jumping back to this step)
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SHEET_DATA_AREA))
{
Expand Down Expand Up @@ -220,6 +218,7 @@ public ActionResult ChangeWorksheet(string sheetIdentifier)
TaskManager.Bus.Remove(EasyUploadTaskManager.VERIFICATION_HEADERFIELDS);
TaskManager.Bus.Remove(EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS);
TaskManager.Bus.Remove(EasyUploadTaskManager.VERIFICATION_ATTRIBUTESUGGESTIONS);
TaskManager.Bus.Remove(EasyUploadTaskManager.ROWS);
#endregion

#region Generate table for selected sheet
Expand Down Expand Up @@ -355,7 +354,7 @@ public ActionResult SelectedAreaToBus()
JsonTableGenerator EUEReader = new JsonTableGenerator(fis);

//Get the worksheet uris and save them to the model
model.SheetUriDictionary = EUEReader.GetWorksheetUris();
model.SheetUriDictionary = EUEReader.GetWorksheetUris();
}
catch (Exception ex)
{
Expand Down
2 changes: 1 addition & 1 deletion Console/BExIS.Web.Shell/Areas/DCM/Controllers/EasyUploadSheetDataStructureController.cs
View file
Expand Up @@ -49,7 +49,7 @@ public ActionResult SheetDataStructure(int index)
if (TaskManager != null)
{
// set SHEET_FORMAT
TaskManager.AddToBus(EasyUploadTaskManager.SHEET_FORMAT, "LeftRight");
TaskManager.AddToBus(EasyUploadTaskManager.SHEET_FORMAT, "TopDown");
TaskManager.Current().SetValid(true);

if (TaskManager.Current().IsValid())
Expand Down
53 changes: 28 additions & 25 deletions Console/BExIS.Web.Shell/Areas/DCM/Controllers/EasyUploadSummaryController.cs
View file
Expand Up @@ -269,9 +269,12 @@ public List<Error> FinishUpload(EasyUploadTaskManager taskManager)
List<VariableIdentifier> identifiers = new List<VariableIdentifier>(); //Used in Excel reader
//Try to find an exact matching datastructure
Boolean foundReusableDataStructure = false;
List<Tuple<int, string, UnitInfo>> MappedHeaders = (List<Tuple<int, string, UnitInfo>>)TaskManager.Bus[EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS];

List<RowModel> headers = (List<RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS];

//For some reason, MappedHeaders might be in a different order in this list than what's indicated by its IDs - to prevent mismatching, sort the headers
MappedHeaders.Sort((m1, m2) => m1.Item1.CompareTo(m2.Item1));
headers.Sort((m1, m2) => m1.Index.CompareTo(m2.Index));

List<StructuredDataStructure> allDatastructures = dsm.StructuredDataStructureRepo.Get().ToList();
foreach (StructuredDataStructure existingStructure in allDatastructures)
{
Expand All @@ -281,17 +284,17 @@ public List<Error> FinishUpload(EasyUploadTaskManager taskManager)
//the same names (labels), datatypes and units in the correct order
List<Variable> variablesOfExistingStructure = existingStructure.Variables.ToList();
foundReusableDataStructure = true;
if (variablesOfExistingStructure.Count != MappedHeaders.Count)
if (variablesOfExistingStructure.Count != headers.Count)
foundReusableDataStructure = false;
else
{
for (int i = 0; i < variablesOfExistingStructure.Count; i++)
{
Variable exVar = variablesOfExistingStructure.ElementAt(i);
Tuple<int, string, UnitInfo> currentHeader = MappedHeaders.ElementAt(i);
if (exVar.Label != currentHeader.Item2 ||
exVar.Unit.Id != currentHeader.Item3.UnitId ||
exVar.DataAttribute.DataType.Id != currentHeader.Item3.SelectedDataTypeId)
RowModel currentHeader = headers.ElementAt(i);
if (exVar.Label != currentHeader.Name ||
exVar.Unit.Id != currentHeader.SelectedUnit.UnitId ||
exVar.DataAttribute.DataType.Id != currentHeader.SelectedDataType.DataTypeId)
{
foundReusableDataStructure = false;
}
Expand Down Expand Up @@ -373,24 +376,24 @@ public List<Error> FinishUpload(EasyUploadTaskManager taskManager)
XmlElement orderElement = xmldoc.CreateElement("order");

//Sorting necessary to prevent problems when inserting the tuples
MappedHeaders.Sort((head1, head2) => head1.Item1.CompareTo(head2.Item1));
headers.OrderBy(r => r.Index);

var dataTypeRepo = unitOfWork.GetReadOnlyRepository<DataType>();
var unitRepo = unitOfWork.GetReadOnlyRepository<Unit>();
var dataAttributeRepo = unitOfWork.GetReadOnlyRepository<DataAttribute>();

List<DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList();

foreach (Tuple<int, string, UnitInfo> Entry in MappedHeaders)
foreach (RowModel header in headers)
{
int i = MappedHeaders.IndexOf(Entry);
int i = headers.IndexOf(header);

DataType dataType = dataTypeRepo.Get(Entry.Item3.SelectedDataTypeId);
Unit CurrentSelectedUnit = unitRepo.Get(Entry.Item3.UnitId);
DataType dataType = dataTypeRepo.Get(header.SelectedDataType.DataTypeId);
Unit CurrentSelectedUnit = unitRepo.Get(header.SelectedUnit.UnitId);

DataAttribute CurrentDataAttribute = new DataAttribute();
//If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match)
DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(Entry.Item2).ToLower()) &&
DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(header.SelectedDataAttribute.Name).ToLower()) &&
da.DataType.Id == dataType.Id &&
da.Unit.Id == CurrentSelectedUnit.Id).FirstOrDefault();
if (existingDataAttribute != null)
Expand All @@ -400,10 +403,10 @@ public List<Error> FinishUpload(EasyUploadTaskManager taskManager)
else
{
//No matching DataAttribute => Create a new one
CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(Entry.Item2), Entry.Item2, "", false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null);
CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(header.Name), header.Name, "", false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null);
}

Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, Entry.Item2, "", "", "");
Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, header.Name, "", "", "");
VariableIdentifier vi = new VariableIdentifier
{
name = newVariable.Label,
Expand Down Expand Up @@ -431,7 +434,7 @@ public List<Error> FinishUpload(EasyUploadTaskManager taskManager)

long datasetId = ds.Id;
long sdsId = sds.Id;

if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault()))
{
DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId);
Expand Down Expand Up @@ -662,8 +665,8 @@ private List<Error> ValidateRows(string JsonArray)
string[][] DeserializedJsonArray = JsonConvert.DeserializeObject<string[][]>(JsonArray);

List<Error> ErrorList = new List<Error>();
List<Tuple<int, string, UnitInfo>> MappedHeaders = (List<Tuple<int, string, UnitInfo>>)TaskManager.Bus[EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS];
Tuple<int, string, UnitInfo>[] MappedHeadersArray = MappedHeaders.ToArray();
List<RowModel> Rows = (List<RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS];
RowModel[] MappedRowsArray = Rows.ToArray();


List<string> DataArea = (List<string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA];
Expand All @@ -686,17 +689,17 @@ private List<Error> ValidateRows(string JsonArray)
int SelectedX = x - (IntDataArea[1]);
string vv = DeserializedJsonArray[y][x];

Tuple<int, string, UnitInfo> mappedHeader = MappedHeaders.Where(t => t.Item1 == SelectedX).FirstOrDefault();
RowModel mappedHeader = MappedRowsArray.Where(t => t.Index == SelectedX).FirstOrDefault();

DataType datatype = null;

if (mappedHeader.Item3.SelectedDataTypeId == -1)
if (mappedHeader.SelectedDataType.DataTypeId == -1)
{
datatype = dtm.Repo.Get(mappedHeader.Item3.DataTypeInfos.FirstOrDefault().DataTypeId);
datatype = dtm.Repo.Get(mappedHeader.SelectedUnit.DataTypeInfos.FirstOrDefault().DataTypeId);
}
else
{
datatype = dtm.Repo.Get(mappedHeader.Item3.SelectedDataTypeId);
datatype = dtm.Repo.Get(mappedHeader.SelectedDataType.DataTypeId);
}

string datatypeName = datatype.SystemType;
Expand All @@ -708,16 +711,16 @@ private List<Error> ValidateRows(string JsonArray)
{
if (vv.Contains("."))
{
dtc = new DataTypeCheck(mappedHeader.Item2, datatypeName, DecimalCharacter.point);
dtc = new DataTypeCheck(mappedHeader.SelectedDataAttribute.Name, datatypeName, DecimalCharacter.point);
}
else
{
dtc = new DataTypeCheck(mappedHeader.Item2, datatypeName, DecimalCharacter.comma);
dtc = new DataTypeCheck(mappedHeader.SelectedDataAttribute.Name, datatypeName, DecimalCharacter.comma);
}
}
else
{
dtc = new DataTypeCheck(mappedHeader.Item2, datatypeName, DecimalCharacter.point);
dtc = new DataTypeCheck(mappedHeader.SelectedDataAttribute.Name, datatypeName, DecimalCharacter.point);
}

var ValidationResult = dtc.Execute(vv, y);
Expand Down

0 comments on commit 9c5d8a1

Please sign in to comment.