Skip to content

Commit

Permalink
Merge branch 'hotfix/sprint-3-fix-feedback-bug' into 'modis_master'
Browse files Browse the repository at this point in the history 
#89 - Fix bug feedback sprint 3

See merge request datalake/docker_datalake!94
  • Loading branch information
latamen.aitmeddour committed Jul 29, 2021
2 parents 814677c + 2fe6f01 commit b546136
Show file tree
Hide file tree
Showing 9 changed files with 253 additions and 77 deletions.
10 changes: 5 additions & 5 deletions service_zone/backend/flask/neocampus/routers/mongo_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,15 +101,15 @@ def get_handled_data_list():
# If there is Influx data (> 1 because Header is present at minimum in csv file) and filter "Time series" is selected
if number_of_rows_influxdb > 1 and "csv" in params.get('filetype'):
metadata_influx_file = {
'filename': 'InfluxDB.csv',
'filename': 'donnees-serie-temporelle-influxdb.csv',
'filesize': sys.getsizeof(influxdb_result)
}
result['influxDB'] = metadata_influx_file

# If there is Mongo data and filter "Images" or "Time series" are selected
if mongo_nb_results > 0 and ("csv" in params.get('filetype') or "image" in params.get('filetype')):
if mongo_nb_results > 0 and "image" in params.get('filetype'):
metadata_mongo_file = {
'filename': 'MongoDB.json',
'filename': 'metadonnees-images-mongodb.json',
'filesize': sys.getsizeof(mongoDB)
}
result['MongoDB'] = metadata_mongo_file
Expand Down Expand Up @@ -160,7 +160,7 @@ def get_handled_data_zipped_file():

# JSON FILE - MONGODB
if(mongo_nb_results > 0):
data = zipfile.ZipInfo("MongoDB.json")
data = zipfile.ZipInfo("metadonnees-images-mongodb.json")
# Datetime
data.date_time = time.localtime(time.time())[:6]

Expand All @@ -172,7 +172,7 @@ def get_handled_data_zipped_file():

# CSV FILE - INFLUXDB
if(number_of_rows_influxdb > 1):
data = zipfile.ZipInfo("InfluxDB.csv")
data = zipfile.ZipInfo("donnees-serie-temporelle-influxdb.csv")
data.date_time = time.localtime(time.time())[:6]
data.compress_type = zipfile.ZIP_DEFLATED

Expand Down
14 changes: 0 additions & 14 deletions service_zone/backend/flask/neocampus/services/mongo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,20 +152,6 @@ def get_handled_data(params):
# Query result (Cursor object)
result_query = collection_traitement_historique.find({'creation_date': {'$gte': start, '$lt': end}},{"content_image":False})

# Time series CSV -> data_historique database
if("csv" in params.get('filetype')):
# Database "data_historique"
mongo_database = mongo_client.data_historique

# Collection "traitement_historique"
collection_name = "traitement_historique"

# Collection "traitement_historique"
collection_traitement_historique = mongo_database[collection_name]

# Query result (Cursor object)
result_query = collection_traitement_historique.find({'creation_date': {'$gte': start, '$lt': end}})

if(collection_name == ""):
return {}, 0

Expand Down
116 changes: 67 additions & 49 deletions service_zone/frontend/datalake-react-front/src/components/Home.js
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,68 @@
import React from "react";

export class Home extends React.Component {
render() {
return(
<div>
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container-fluid">
<a class="navbar-brand" href="/">Datalake</a>

<div>
</div>
<a class="btn btn-outline-primary" href="/logout" role="button">Logout</a>
</div>
</nav>
<div class="p-4">
<div class="row">
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Download</h5>
<p class="card-text">Partie téléchargement des données brutes et traitées du Datalake.</p>
<a href="/download" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Upload</h5>
<p class="card-text">Envoi des données vers le Datalake.</p>
<a href="/upload" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Datavisualisation data processed Time Series</h5>
<p class="card-text">Partie visualisation des données traitées time series.</p>
<a href="/data-processed-visualisation" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
</div>
</div>
</div>
);
}
import React from "react";

export class Home extends React.Component {
render() {
return(
<div>
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container-fluid">
<a class="navbar-brand" href="/">Datalake</a>

<div>
</div>
<a class="btn btn-outline-primary" href="/logout" role="button">Logout</a>
</div>
</nav>
<div class="p-4">
<div class="row">
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Data Visualization</h5>
<p class="card-text">Partie data visualization des données traitées du Datalake.</p>
<a href="/data-visualization" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Download Raw Data</h5>
<p class="card-text">Téléchargement des données brutes du Datalake.</p>
<a href="/download" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Upload</h5>
<p class="card-text">Envoi des données vers le Datalake.</p>
<a href="/upload" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Datavisualisation data processed Time Series</h5>
<p class="card-text">Partie visualisation des données traitées time series.</p>
<a href="/data-processed-visualisation" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
<div class="col-sm-4">
<div class="card border-dark mb-3">
<div class="card-body">
<h5 class="card-title">Download Processed Data</h5>
<p class="card-text">Téléchargement des données traitées du Datalake.</p>
<a href="/download-handled-data" class="btn btn-outline-primary">Accéder</a>
</div>
</div>
</div>
</div>
</div>
</div>
);
}
}
2 changes: 1 addition & 1 deletion service_zone/frontend/datalake-react-front/src/components/Upload.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ export class Upload extends React.Component {
</div>
<Metadonnees />
<div class="form-group required">
<Dropzone value={this.state.file} name="file" onDrop={this.onDrop} accept="image/*,application/JSON,.csv,text/plain,.sql">
<Dropzone value={this.state.file} name="file" onDrop={this.onDrop} accept="image/*,application/JSON,.csv,text/plain,.sql,application/x-gzip,application/x-zip-compressed">
{({getRootProps, getInputProps}) => (
<section>
<div {...getRootProps({className: 'drop'})}>
Expand Down
15 changes: 11 additions & 4 deletions .../frontend/datalake-react-front/src/components/download-handled-data/DownloadHandleData.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,11 @@ export class DownloadHandleData extends React.Component {
let body1 = []
let json_object = {}
selectedElements.map(element => {
if(element.filename == "MongoDB.json"){
if(element.filename == "metadonnees-images-mongodb.json"){
json_object.mongodb_file = true
}

if(element.filename == "InfluxDB.csv"){
if(element.filename == "donnees-serie-temporelle-influxdb.csv"){
json_object.influxdb_file = true
}
})
Expand All @@ -152,7 +152,7 @@ export class DownloadHandleData extends React.Component {
if(selectedElements.length) {
this.handleShow();
api.post('handled-data-file', body, {
'responseType': 'arraybuffer'
responseType: 'arraybuffer'
})
.then(function (result) {
const url = window.URL.createObjectURL(new Blob([result.data], {type: 'application/zip'}));
Expand Down Expand Up @@ -186,6 +186,8 @@ export class DownloadHandleData extends React.Component {
'beginDate': this.state.beginDate,
'endDate': this.state.endDate
}
let beginDate = this.state.beginDate
let endDate = this.state.endDate

return (
<div>
Expand All @@ -206,6 +208,8 @@ export class DownloadHandleData extends React.Component {
<tr>
<th scope="col">Nom du fichier</th>
<th scope="col">Taille (en bytes)</th>
<th scope="col">Date de début</th>
<th scope="col">Date de fin</th>
</tr>
</thead>
<tbody>
Expand All @@ -215,7 +219,10 @@ export class DownloadHandleData extends React.Component {
Object.keys(elts).map(function(key, index){
return <RowItem key={index} item={elts[key]}
handler={handler}
selectedElements={selectedElements} />
selectedElements={selectedElements}
beginDate={beginDate}
endDate={endDate}
/>

}) }

Expand Down
5 changes: 4 additions & 1 deletion service_zone/frontend/datalake-react-front/src/components/download-handled-data/RowItem.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,14 @@ export class RowItem extends React.Component {
}

render() {

let beginDate = this.props.beginDate
let endDate = this.props.endDate
return(
<tr>
<td scope="row">{ this.props.item.filename }</td>
<td>{ this.props.item.filesize }</td>
<td>{ beginDate }</td>
<td>{ endDate }</td>
<td>
<div class="form-check">
<input class="form-check-input" onChange={this.handleChange} checked={this.isSelected()} type="checkbox" value="" id="flexCheckDefault" />
Expand Down
6 changes: 5 additions & 1 deletion service_zone/frontend/datalake-react-front/src/components/download-raw-data/Filters.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import React from "react";
import { FormGroup, FormLabel, Form, Button } from "react-bootstrap";
import { config } from '../../configmeta/config';
import { config_processed_data } from '../../configmeta/config_processed_data';

export class Filters extends React.Component {

Expand Down Expand Up @@ -77,7 +78,10 @@ export class Filters extends React.Component {
render() {
// data type field
const SelectDatatype = () => {
const types = [config.types];
let types = [config.types];
if(this.props.title == "Affichage des données traitées"){
types = [config_processed_data.types];
}
// loop into conf to get all data types
const listTypes = types.map((type) => (
type.map((t) =>
Expand Down
4 changes: 2 additions & 2 deletions service_zone/frontend/datalake-react-front/src/configmeta/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export const config = {
},
{
"id": 1,
"label": "Texte",
"label": "Texte / CSV / Dump SQL",
"type_file_accepted": ["text/plain", "application/csv", "application/vnd.ms-excel", "application/sql"],
"metadonnees": [
{
Expand Down Expand Up @@ -156,7 +156,7 @@ export const config = {
},
{
"id": 8,
"label": "Zip",
"label": "Archive",
"type_file_accepted": ["application/x-zip-compressed", "application/x-gzip"],
"metadonnees": [
{
Expand Down
Loading

0 comments on commit b546136

Please sign in to comment.