Skip to content

Commit

Permalink
Add create missing usage option for upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@fdaugan
fdaugan committed Nov 21, 2021
1 parent 27f1c2e commit b566341
Show file tree
Hide file tree
Showing 7 changed files with 79 additions and 31 deletions.
12 changes: 6 additions & 6 deletions src/main/java/org/ligoj/app/plugin/prov/AbstractProvQuoteVmResource.java
View file
Expand Up @@ -65,7 +65,7 @@ public abstract class AbstractProvQuoteVmResource<T extends AbstractInstanceType
/**
* The default usage : 100% for 1 month.
*/
protected static final ProvUsage USAGE_DEFAULT = new ProvUsage();
public static final ProvUsage USAGE_DEFAULT = new ProvUsage();

/**
* The default budget : no initial cost.
Expand Down Expand Up @@ -273,9 +273,9 @@ protected UpdatedCost delete(final int id) {
/**
* Return the resolved usage entity from it's name.
*
* @param configuration Configuration containing the default values.
* @param name The usage name.
* @return The resolved usage entity. Never <code>null</code> since the configurtion's usage or else
* @param configuration Configuration containing the default values and defined usages.
* @param name The usage name to resolve.
* @return The resolved usage entity. Never <code>null</code> since the configuration's usage or else
* {@link #USAGE_DEFAULT} is used as default value.
*/
protected ProvUsage getUsage(final ProvQuote configuration, final String name) {
Expand All @@ -290,8 +290,8 @@ protected ProvUsage getUsage(final ProvQuote configuration, final String name) {
* Return the resolved budget entity from it's name.
*
* @param configuration Configuration containing the default values.
* @param name The usage name.
* @return The resolved usage entity. Never <code>null</code> since the configurtion's budget or else
* @param name The budget name to resolve.
* @return The resolved b entity. Never <code>null</code> since the configuration's budget or else
* {@link #BUDGET_DEFAULT} is used as default value.
*/
protected ProvBudget getBudget(final ProvQuote configuration, final String name) {
Expand Down
61 changes: 47 additions & 14 deletions src/main/java/org/ligoj/app/plugin/prov/quote/upload/ProvQuoteUploadResource.java
View file
Expand Up @@ -45,6 +45,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.cxf.jaxrs.ext.multipart.Multipart;
import org.hibernate.Hibernate;
import org.ligoj.app.plugin.prov.AbstractProvQuoteVmResource;
import org.ligoj.app.plugin.prov.AbstractQuoteVmEditionVo;
import org.ligoj.app.plugin.prov.ProvResource;
import org.ligoj.app.plugin.prov.ProvTagResource;
Expand All @@ -56,6 +57,7 @@
import org.ligoj.app.plugin.prov.model.ProvQuoteDatabase;
import org.ligoj.app.plugin.prov.model.ProvQuoteInstance;
import org.ligoj.app.plugin.prov.model.ProvQuoteStorage;
import org.ligoj.app.plugin.prov.model.ProvUsage;
import org.ligoj.app.plugin.prov.model.ResourceType;
import org.ligoj.app.plugin.prov.quote.database.ProvQuoteDatabaseResource;
import org.ligoj.app.plugin.prov.quote.database.QuoteDatabaseEditionVo;
Expand Down Expand Up @@ -332,7 +334,7 @@ public void upload(final int subscription, final InputStream uploadedFile, final
final boolean headersIncluded, final String usage, final Integer ramMultiplier, final String encoding,
final String separator) throws IOException {
upload(subscription, uploadedFile, headers, headersIncluded, usage, MergeMode.KEEP, ramMultiplier, encoding,
separator);
false, separator);
}

/**
Expand All @@ -344,11 +346,12 @@ public void upload(final int subscription, final InputStream uploadedFile, final
* @param headers the CSV header names. When <code>null</code> or empty, the default headers are used.
* @param headersIncluded When <code>true</code>, the first line is the headers and the given <code>headers</code>
* parameter is ignored. Otherwise the <code>headers</code> parameter is used.
* @param usage The optional usage name. When not <code>null</code>, each quote instance will be
* associated to this usage.
* @param defaultUsage The optional usage name. When not <code>null</code>, each quote instance without defined
* usage will be associated to this usage.
* @param mode The merge option indicates how the entries are inserted.
* @param ramMultiplier The multiplier for imported RAM values. Default is 1.
* @param encoding CSV encoding. Default is UTF-8.
* @param createUsage When <code>true</code>, missing usage are automatically created.
* @param separator CSV separator. Default is ";".
* @throws IOException When the CSV stream cannot be written.
*/
Expand All @@ -359,10 +362,11 @@ public void upload(@PathParam("subscription") final int subscription,
@Multipart(value = CSV_FILE) final InputStream uploadedFile,
@Multipart(value = "headers", required = false) final String[] headers,
@Multipart(value = "headers-included", required = false) final boolean headersIncluded,
@Multipart(value = "usage", required = false) final String usage,
@Multipart(value = "usage", required = false) final String defaultUsage,
@Multipart(value = "mergeUpload", required = false) final MergeMode mode,
@Multipart(value = "memoryUnit", required = false) final Integer ramMultiplier,
@Multipart(value = "encoding", required = false) final String encoding,
@Multipart(value = "createMissingUsage", required = false) final boolean createUsage,
@Multipart(value = "separator", required = false) final String separator) throws IOException {
log.info("Upload provisioning requested...");
subscriptionResource.checkVisible(subscription);
Expand Down Expand Up @@ -410,7 +414,7 @@ public void upload(@PathParam("subscription") final int subscription,
context.previousQb = previousQb;
list.stream().filter(Objects::nonNull).filter(i -> i.getName() != null).forEach(i -> {
try {
persist(subscription, usage, mode, ramMultiplier, list, cursor, context, i);
persist(subscription, defaultUsage, mode, ramMultiplier, list.size(), cursor, context, createUsage, i);
} catch (final ValidationJsonException e) {
throw handleValidationError(i, e);
} catch (final ConstraintViolationException e) {
Expand All @@ -424,7 +428,7 @@ public void upload(@PathParam("subscription") final int subscription,
}

private <V extends AbstractQuoteVmEditionVo> V copy(final VmUpload upload, final int subscription,
final String usage, final Integer ramMultiplier, final V vo) {
final String defaultUsage, final Integer ramMultiplier, final V vo) {
// Validate the upload object
vo.setName(upload.getName());
vo.setDescription(upload.getDescription());
Expand All @@ -442,7 +446,8 @@ private <V extends AbstractQuoteVmEditionVo> V copy(final VmUpload upload, final
vo.setConstant(upload.getConstant());
vo.setPhysical(upload.getPhysical());
vo.setUsage(Optional.ofNullable(upload.getUsage())
.map(u -> resource.findConfiguredByName(usageRepository, u, subscription).getName()).orElse(usage));
.map(u -> resource.findConfiguredByName(usageRepository, u, subscription).getName())
.orElse(defaultUsage));
vo.setRam(
ObjectUtils.defaultIfNull(ramMultiplier, 1) * ObjectUtils.defaultIfNull(upload.getRam(), 0).intValue());
vo.setSubscription(subscription);
Expand Down Expand Up @@ -482,26 +487,54 @@ private QuoteDatabaseEditionVo newDatabaseVo(final VmUpload upload) {
return vo;
}

private void persist(final int subscription, final String usage, final MergeMode mode, final Integer ramMultiplier,
final List<VmUpload> list, final AtomicInteger cursor, final UploadContext context, VmUpload i) {
private void persist(final int subscription, final String defaultUsage, final MergeMode mode,
final Integer ramMultiplier, final int size, final AtomicInteger cursor, final UploadContext context,
final boolean createUsage, final VmUpload i) {
// Normalize the usage
if (i.getUsage() != null) {
var usage = context.quote.getUsages().stream().filter(u -> u.getName().equalsIgnoreCase(i.getUsage()))
.findFirst().orElse(null);
if (usage != null && createUsage) {
// Normalize the name as needed
i.setUsage(usage.getName());
} else if (createUsage) {
// Create the missing usage
usage = new ProvUsage();
usage.setName(i.getUsage());
usage.setRate(AbstractProvQuoteVmResource.USAGE_DEFAULT.getRate());
usage.setConfiguration(context.quote);
usageRepository.saveAndFlush(usage);
context.quote.getUsages().add(usage);
}
}

if (StringUtils.isNotEmpty(i.getEngine())) {
// Database case
final var merger = mergersDatabase.get(ObjectUtils.defaultIfNull(mode, MergeMode.KEEP));
final var vo = copy(i, subscription, usage, ramMultiplier, newDatabaseVo(i));
final var vo = copy(i, subscription, defaultUsage, ramMultiplier, newDatabaseVo(i));
vo.setPrice(
qbResource.validateLookup("database", qbResource.lookup(context.quote, vo), vo.getName()).getId());
persist(i, subscription, merger, context, vo, QuoteStorageEditionVo::setDatabase, ResourceType.DATABASE);
} else {
// Instance/Container case
final var merger = mergersInstance.get(ObjectUtils.defaultIfNull(mode, MergeMode.KEEP));
final var vo = copy(i, subscription, usage, ramMultiplier, newInstanceVo(i));
final var vo = copy(i, subscription, defaultUsage, ramMultiplier, newInstanceVo(i));
vo.setPrice(
qiResource.validateLookup("instance", qiResource.lookup(context.quote, vo), vo.getName()).getId());
persist(i, subscription, merger, context, vo, QuoteStorageEditionVo::setInstance, ResourceType.INSTANCE);
}
final var percent = ((int) (cursor.incrementAndGet() * 100D / list.size()));
if (cursor.get() > 1 && percent / 10 > ((int) ((cursor.get() - 1) * 100D / list.size())) / 10) {
log.info("Upload provisioning : importing {} entries, {}%", list.size(), percent);

// Update the cursor
increment(cursor, size);
}

/**
* Handle cursor and progress display.
*/
private void increment(final AtomicInteger cursor, final int size) {
final var percent = ((int) (cursor.incrementAndGet() * 100D / size));
if (cursor.get() > 1 && percent / 10 > ((int) ((cursor.get() - 1) * 100D / size)) / 10) {
log.info("Upload provisioning : importing {} entries, {}%", size, percent);
}
}

Expand Down
12 changes: 7 additions & 5 deletions src/main/resources/META-INF/resources/webjars/service/prov/nls/fr/messages.js
View file
Expand Up @@ -100,12 +100,12 @@ define({
'service:prov:container-size': 'Taille',
'service:prov:containers-block': 'Conteneurs',
'service:prov:function': 'Fonction',
'service:prov:function-requests-help' : 'Nombre d\'invocations (en millions)de cette fonctin durant un mois',
'service:prov:function-requests-help': 'Nombre d\'invocations (en millions)de cette fonctin durant un mois',
'service:prov:function-duration-help': 'Durée maximale (en milli secondes) de cette fonction',
'service:prov:function-concurrency-help': 'Concurence moyenne de cette fonction. Difficile à renseigner, et devrait correspondre au percentile p99 et non pas à une réelle moyenne. Peut être inférieure à 1',
'service:prov:function-requests' : 'Requêtes',
'service:prov:function-millions' : 'Millions',
'service:prov:function-milliseconds' : 'Millisecondes',
'service:prov:function-requests': 'Requêtes',
'service:prov:function-millions': 'Millions',
'service:prov:function-milliseconds': 'Millisecondes',
'service:prov:function-duration': 'Durée',
'service:prov:function-concurrency': 'Concurence',
'service:prov:functions-block': 'Fonctions',
Expand Down Expand Up @@ -161,7 +161,7 @@ define({
'service:prov:storage-rate-good': 'Bon',
'service:prov:storage-rate-best': 'Meilleur',
'service:prov:resources': 'Ressources',
'service:prov:tag-help':'Tags des ressources',
'service:prov:tag-help': 'Tags des ressources',
'service:prov:total': 'Total',
'service:prov:total-ram': 'Mémoire totale',
'service:prov:total-cpu': 'CPU total',
Expand Down Expand Up @@ -239,6 +239,8 @@ define({
'csv-headers-included': 'CSV contient les entêtes',
'csv-headers': 'Entêtes',
'csv-headers-included-help': 'Lorsque les entêtes sont en première ligne du fichier',
'csv-create-missing-usage': 'Créer profil d\'utilisation',
'csv-create-missing-usage-help': 'Créer un profil d\'utilisation non existant au lieu de provoquer une erreur',
'csv-separator': 'Séparateur',
'csv-separator-help': 'Caractère de séparation des champs du CSV',
'error': {
Expand Down
6 changes: 4 additions & 2 deletions src/main/resources/META-INF/resources/webjars/service/prov/nls/messages.js
View file
Expand Up @@ -212,7 +212,7 @@ define({
'service:prov:usage-null': 'Always used',
'service:prov:usage-help': 'Chosen usage will infer the term, and the best cost. Available usages are at the subscription level. When undefined, the default usage is used. And when there is no default usage, it will be 100% for one month.',
'service:prov:usage': 'Usage profile',
'service:prov:usage-upload-help': 'Usage to associate to each imported entry',
'service:prov:usage-upload-help': 'Default usage to associate to each imported entry when undefined',
'service:prov:usage-default': 'Default usage rate : {{this}}%',
'service:prov:usage-actual-cost': 'Actual usage rate : {{this}}%',
'service:prov:usage-partial': 'Use only {{[0]}} of {{[1]}} available ({{[2]}}%)',
Expand All @@ -227,7 +227,7 @@ define({
'service:prov:budget-help': 'Chosen budget will infer the term, and the best cost. Available budgets are at the subscription level. When undefined, the default budget is used. And when there is no default budget, it will be considered with no cash available.',
'service:prov:budget': 'Budget profile',
'service:prov:budget-null': 'No cash',
'service:prov:budget-upload-help': 'Budget profile to associate to each imported entry',
'service:prov:budget-upload-help': 'Default budget profile to associate to each imported entry when undefined',
'service:prov:budget-default': 'Default budget : {{this}}',
'service:prov:budget-initialCost': 'Available cash:',
'service:prov:budget-initialCost-help': 'Available cash dedicated to this budget',
Expand All @@ -241,6 +241,8 @@ define({
'csv-headers-included': 'CSV file has headers',
'csv-headers': 'Headers',
'csv-headers-included-help': 'When headers are in the first line of CSV file',
'csv-create-missing-usage': 'Create usage',
'csv-create-missing-usage-help': 'Create non existing usage instead of generating an error',
'csv-separator': 'Separator',
'csv-separator-help': 'Separator character of CSV fields',
'error': {
Expand Down
10 changes: 10 additions & 0 deletions src/main/resources/META-INF/resources/webjars/service/prov/prov.html
View file
Expand Up @@ -824,6 +824,16 @@ <h4 class="modal-title" id="import-modal-title">{{upload-new}}</h4>
</div>
<input type="text" id="instance-usage-upload-name" class="hidden" name="usage">
</div>
<div class="form-group">
<div class="col-sm-9 col-sm-offset-3">
<div class="checkbox">
<label for="csv-create-missing-usage">
<input type="checkbox" checked id="csv-create-missing-usage" name="createMissingUsage"> {{csv-create-missing-usage}}
</label>
</div>
<p class="help-block">{{csv-create-missing-usage-help}}</p>
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-3" for="instance-budget-upload" data-toggle="tooltip" title="{{service:prov:budget-help}}">{{service:prov:budget}}</label>
<div class="col-sm-9">
Expand Down
3 changes: 2 additions & 1 deletion src/main/resources/META-INF/resources/webjars/service/prov/prov.js
View file
Expand Up @@ -1401,7 +1401,7 @@ define(function () {

var current = {

$messages = {},
$messages: {},

/**
* Current quote.
Expand Down Expand Up @@ -1976,6 +1976,7 @@ define(function () {
_('instance-usage-upload-name').val((_('instance-usage-upload').select2('data') || {}).name || null);
_('instance-budget-upload-name').val((_('instance-budget-upload').select2('data') || {}).name || null);
_('csv-headers-included').val(_('csv-headers-included').is(':checked') ? 'true' : 'false');
_('csv-create-missing-usage').val(_('csv-create-missing-usage').is(':checked') ? 'true' : 'false');
$popup.find('input[type="text"]').not('[readonly]').not('.select2-focusser').not('[disabled]').filter(function () {
return $(this).val() === '';
}).attr('disabled', 'disabled').attr('readonly', 'readonly').addClass('temp-disabled').closest('.select2-container').select2('enable', false);
Expand Down
6 changes: 3 additions & 3 deletions ...test/java/org/ligoj/app/plugin/prov/quote/upload/ProvQuoteInstanceUploadResourceTest.java
View file
Expand Up @@ -196,7 +196,7 @@ void uploadDatabaseUpdate() throws IOException {
IOUtils.toInputStream("database4;0.5;1000;oracle;standard two\ndatabaseNEW;0.4;800;mysql;",
DEFAULT_ENCODING),
new String[] { "name", "cpu", "ram", "engine", "edition" }, false, "Full Time 12 month",
MergeMode.UPDATE, 1, DEFAULT_ENCODING, DEFAULT_SEPARATOR);
MergeMode.UPDATE, 1, DEFAULT_ENCODING, false, DEFAULT_SEPARATOR);
var configuration = getConfiguration();
checkCost(configuration.getCost(), 4905.058, 7354.658, false);
configuration = getConfiguration();
Expand Down Expand Up @@ -383,7 +383,7 @@ void uploadTagsInvalidTagName() throws IOException {
void uploadUpdate() throws IOException {
qiuResource.upload(subscription,
IOUtils.toInputStream("ANY;0.5;500;LINUX\nANY 1;1;2000;LINUX\nANY;2;1000;LINUX", DEFAULT_ENCODING),
new String[] { "name", "cpu", "ram", "os" }, false, null, MergeMode.UPDATE, 1, DEFAULT_ENCODING,
new String[] { "name", "cpu", "ram", "os" }, false, null, MergeMode.UPDATE, 1, DEFAULT_ENCODING, false,
ProvQuoteUploadResource.DEFAULT_SEPARATOR);
final var configuration = getConfiguration();
Assertions.assertEquals(9, configuration.getInstances().size());
Expand Down Expand Up @@ -413,7 +413,7 @@ void uploadConflictName() throws IOException {
final var input = IOUtils.toInputStream("ANY;0.5;500;LINUX\nANY;2;1000;LINUX", DEFAULT_ENCODING);
Assertions.assertThrows(DataIntegrityViolationException.class,
() -> qiuResource.upload(subscription, input, new String[] { "name", "cpu", "ram", "os" }, false, null,
MergeMode.INSERT, 1, DEFAULT_ENCODING, ProvQuoteUploadResource.DEFAULT_SEPARATOR));
MergeMode.INSERT, 1, DEFAULT_ENCODING, false, ProvQuoteUploadResource.DEFAULT_SEPARATOR));
}

@Test
Expand Down

0 comments on commit b566341

Please sign in to comment.