Skip to content

Commit

Permalink
Merge pull request #677 from Laixer/develop
Browse files Browse the repository at this point in the history
3.6.4
  • Loading branch information
yorickdewid committed Sep 6, 2022
2 parents 0412b39 + 98607d2 commit fba60fa
Show file tree
Hide file tree
Showing 13 changed files with 205 additions and 338 deletions.
151 changes: 9 additions & 142 deletions database/fundermaps_base.sql

Large diffs are not rendered by default.

14 changes: 0 additions & 14 deletions database/precompute.sql
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,6 @@ SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;

--
-- Data for Name: building_type; Type: TABLE DATA; Schema: data; Owner: fundermaps
--

REFRESH MATERIALIZED VIEW "data".building_type WITH DATA;


--
-- Data for Name: analysis_foundation_risk; Type: TABLE DATA; Schema: data; Owner: fundermaps
--

REFRESH MATERIALIZED VIEW "data".analysis_foundation_risk WITH DATA;


--
-- Data for Name: analysis_complete; Type: TABLE DATA; Schema: data; Owner: fundermaps
--
Expand Down
2 changes: 1 addition & 1 deletion src/FunderMaps.AspNetCore/FunderMaps.AspNetCore.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

<ItemGroup>
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.5" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.7" />
</ItemGroup>

<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion src/FunderMaps.Core/FunderMaps.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="6.0.0" />
<PackageReference Include="Scriban" Version="5.4.5" />
<PackageReference Include="Scriban" Version="5.5.0" />
</ItemGroup>

</Project>
17 changes: 8 additions & 9 deletions src/FunderMaps.Core/Interfaces/IAppContextFactory.cs
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
namespace FunderMaps.Core.Interfaces
namespace FunderMaps.Core.Interfaces;

/// <summary>
/// AppContext factory.
/// </summary>
public interface IAppContextFactory
{
/// <summary>
/// AppContext factory.
/// Create the <see cref="AppContext"/>.
/// </summary>
public interface IAppContextFactory
{
/// <summary>
/// Create the <see cref="AppContext"/>.
/// </summary>
AppContext Create();
}
AppContext Create();
}
239 changes: 119 additions & 120 deletions src/FunderMaps.Core/MapBundle/Jobs/ExportJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;

namespace FunderMaps.Core.MapBundle.Jobs
namespace FunderMaps.Core.MapBundle.Jobs;

/// <summary>
/// Bundle job entry.
/// </summary>
public class ExportJob : CommandTask
{
/// <summary>
/// Bundle job entry.
/// </summary>
public class ExportJob : CommandTask
{
private const string TaskName = "BUNDLE_BUILDING";
private const string TaskName = "BUNDLE_BUILDING";

private static readonly string[] exportLayers = new[]
{
private static readonly string[] exportLayers = new[]
{
"analysis_building",
"analysis_foundation",
"analysis_quality",
Expand All @@ -24,142 +24,141 @@ public class ExportJob : CommandTask
"incident_aggregate_category",
};

private readonly string connectionString;
private readonly string connectionString;

private readonly IMapService _mapService;
private readonly IBlobStorageService _blobStorageService;

/// <summary>
/// Create new instance.
/// </summary>
public ExportJob(IConfiguration configuration, IMapService mapService, IBlobStorageService blobStorageService, ILogger<ExportJob> logger)
{
Logger = logger;
connectionString = configuration.GetConnectionString("FunderMapsConnection");
_mapService = mapService;
_blobStorageService = blobStorageService;
}

private readonly IMapService _mapService;
private readonly IBlobStorageService _blobStorageService;
/// <summary>
/// Run the background command.
/// </summary>
/// <remarks>
/// Select the export layers in random order.
/// </remarks>
/// <param name="context">Command task execution context.</param>
public override async Task ExecuteCommandAsync(CommandTaskContext context)
{
Context = context;
var rng = new Random();

/// <summary>
/// Create new instance.
/// </summary>
public ExportJob(IConfiguration configuration, IMapService mapService, IBlobStorageService blobStorageService, ILogger<ExportJob> logger)
// Remove any previous artefacts.
if (Directory.Exists(context.Workspace))
{
Logger = logger;
connectionString = configuration.GetConnectionString("FunderMapsConnection");
_mapService = mapService;
_blobStorageService = blobStorageService;
Directory.Delete(context.Workspace, true);
}

/// <summary>
/// Run the background command.
/// </summary>
/// <remarks>
/// Select the export layers in random order.
/// </remarks>
/// <param name="context">Command task execution context.</param>
public override async Task ExecuteCommandAsync(CommandTaskContext context)
// var layer = exportLayers.OrderBy(i => rng.Next()).First();
foreach (var layer in exportLayers.OrderBy(i => rng.Next()))
{
Context = context;
var rng = new Random();
Logger.LogInformation($"Building layer {layer}");

// Remove any previous artefacts.
if (Directory.Exists(context.Workspace))
async Task<FileDataSource> buildGpkg()
{
Directory.Delete(context.Workspace, true);
}
FileDataSource fileOutput = new()
{
Format = GeometryFormat.GeoPackage,
PathPrefix = CreateDirectory("out"),
Extension = ".gpkg",
Name = layer,
};

CommandInfo command = new VectorDatasetBuilder()
.InputDataset(new PostreSQLDataSource(connectionString))
.InputLayers(new BundleLayerSource(layer, Context.Workspace))
.OutputDataset(fileOutput)
.Build(formatName: "GPKG");

if (await RunCommandAsync(command) == 0)
{
Logger.LogDebug("GPKG dataset complete");

// var layer = exportLayers.OrderBy(i => rng.Next()).First();
foreach (var layer in exportLayers.OrderBy(i => rng.Next()))
{
Logger.LogInformation($"Building layer {layer}");
using var fileOutputStream = File.OpenRead(fileOutput.ToString());

async Task<FileDataSource> buildGpkg()
{
FileDataSource fileOutput = new()
{
Format = GeometryFormat.GeoPackage,
PathPrefix = CreateDirectory("out"),
Extension = ".gpkg",
Name = layer,
};

CommandInfo command = new VectorDatasetBuilder()
.InputDataset(new PostreSQLDataSource(connectionString))
.InputLayers(new BundleLayerSource(layer, Context.Workspace))
.OutputDataset(fileOutput)
.Build(formatName: "GPKG");

if (await RunCommandAsync(command) == 0)
{
Logger.LogDebug("GPKG dataset complete");

using var fileOutputStream = File.OpenRead(fileOutput.ToString());

await _blobStorageService.StoreFileAsync(
containerName: "geo-bundle",
fileName: fileOutput.FileName,
contentType: "application/x-sqlite3",
stream: fileOutputStream);

Logger.LogDebug("GPKG upload complete");

return fileOutput;
}

throw new Exception();
}
await _blobStorageService.StoreFileAsync(
containerName: "geo-bundle",
fileName: fileOutput.FileName,
contentType: "application/x-sqlite3",
stream: fileOutputStream);

var bundleFileGpkg = await buildGpkg();
Logger.LogDebug("GPKG upload complete");

async Task<FileDataSource> buildGeoJSON(FileDataSource fileInput)
{
FileDataSource fileOutput = new()
{
Format = GeometryFormat.GeoJSONSeq,
PathPrefix = CreateDirectory("out"),
Extension = ".json",
Name = layer,
};
return fileOutput;
}

CommandInfo command = new VectorDatasetBuilder()
.InputDataset(fileInput)
.OutputDataset(fileOutput)
.Build(formatName: "GeoJSONSeq");
throw new Exception();
}

if (await RunCommandAsync(command) == 0)
{
Logger.LogDebug("GeoJSON dataset complete");
var bundleFileGpkg = await buildGpkg();

using var fileOutputStream = File.OpenRead(fileOutput.ToString());
async Task<FileDataSource> buildGeoJSON(FileDataSource fileInput)
{
FileDataSource fileOutput = new()
{
Format = GeometryFormat.GeoJSONSeq,
PathPrefix = CreateDirectory("out"),
Extension = ".json",
Name = layer,
};

CommandInfo command = new VectorDatasetBuilder()
.InputDataset(fileInput)
.OutputDataset(fileOutput)
.Build(formatName: "GeoJSONSeq");

if (await RunCommandAsync(command) == 0)
{
Logger.LogDebug("GeoJSON dataset complete");

await _blobStorageService.StoreFileAsync(
containerName: "geo-bundle",
fileName: fileOutput.FileName,
contentType: "application/json",
stream: fileOutputStream);
using var fileOutputStream = File.OpenRead(fileOutput.ToString());

Logger.LogDebug("GeoJSON upload complete");
await _blobStorageService.StoreFileAsync(
containerName: "geo-bundle",
fileName: fileOutput.FileName,
contentType: "application/json",
stream: fileOutputStream);

return fileOutput;
}
Logger.LogDebug("GeoJSON upload complete");

throw new Exception();
return fileOutput;
}

var bundleFileGeoJSON = await buildGeoJSON(bundleFileGpkg);
throw new Exception();
}

// NOTE: We *must* delete the old dataset first.
await _mapService.DeleteDatasetAsync(layer);
var bundleFileGeoJSON = await buildGeoJSON(bundleFileGpkg);

if (await _mapService.UploadDatasetAsync(layer, bundleFileGeoJSON.ToString()))
{
Logger.LogDebug("Layer upload complete");
}
// NOTE: We *must* delete the old dataset first.
await _mapService.DeleteDatasetAsync(layer);

if (await _mapService.PublishAsync(layer))
{
Logger.LogInformation($"Layer published");
}
if (await _mapService.UploadDatasetAsync(layer, bundleFileGeoJSON.ToString()))
{
Logger.LogDebug("Layer upload complete");
}
}

/// <summary>
/// Method to check if a task can be handeld by this job.
/// </summary>
/// <param name="name">The task name.</param>
/// <param name="value">The task payload.</param>
/// <returns><c>True</c> if method handles task, false otherwise.</returns>
public override bool CanHandle(string name, object value)
=> name is not null && name.ToUpperInvariant() == TaskName;
if (await _mapService.PublishAsync(layer))
{
Logger.LogInformation($"Layer published");
}
}
}

/// <summary>
/// Method to check if a task can be handeld by this job.
/// </summary>
/// <param name="name">The task name.</param>
/// <param name="value">The task payload.</param>
/// <returns><c>True</c> if method handles task, false otherwise.</returns>
public override bool CanHandle(string name, object value)
=> name is not null && name.ToUpperInvariant() == TaskName;
}
2 changes: 1 addition & 1 deletion src/FunderMaps.Data/FunderMaps.Data.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Npgsql" Version="6.0.4" />
<PackageReference Include="Npgsql" Version="6.0.5" />
</ItemGroup>

<ItemGroup>
Expand Down
Loading

0 comments on commit fba60fa

Please sign in to comment.