Skip to content

Commit

Permalink
Merge branch 'issue1921_solved' into 'master'
Browse files Browse the repository at this point in the history 
Fix issues #19 and #21

Closes #19 and #21

See merge request pml/x_timecluster_extension!13
  • Loading branch information
@luisgasco
luisgasco committed Oct 29, 2020
2 parents 5fe91df + 4e1546a commit fdf596a
Show file tree
Hide file tree
Showing 11 changed files with 4,838 additions and 628 deletions.
4 changes: 2 additions & 2 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ RUN pip install nbdev
RUN pip install umap-learn
RUN pip install tensorflow
RUN pip install keras
RUN pip install papermill
# RUN pip install papermill
RUN pip install seaborn
RUN pip install plotly

## Python packages that need to be upgraded
RUN pip install --upgrade wandb fastcore
RUN pip install --upgrade wandb fastcore papermill

# Environmental variables for wandb
ENV LC_ALL=C.UTF-8
Expand Down
200 changes: 123 additions & 77 deletions nbs/01_dataset_artifact.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 72,
"metadata": {
"hide_input": true
},
Expand All @@ -13,9 +13,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 73,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"%nbdev_hide\n",
"import sys\n",
Expand All @@ -33,7 +42,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 74,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -47,7 +56,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 75,
"metadata": {},
"outputs": [
{
Expand All @@ -71,7 +80,7 @@
" Path('/home/user/data/PACMEL-2019/JNK/mining-variables.csv')]"
]
},
"execution_count": null,
"execution_count": 75,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -83,7 +92,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 76,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -119,7 +128,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 77,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -129,7 +138,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 78,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -138,7 +147,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 79,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -148,7 +157,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -157,28 +166,50 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"ruta_wandb_artifacts = 'work/temp_data/wandb_artifacts'# When permissions are fixed, save artifacts in 'data/PACMEL-2019/wandb_artifacts'\n",
"ar = TSArtifact.from_df(df, 'JNK', normalize=False, path= Path.home()/ruta_wandb_artifacts)"
"#17280 muestras = 1 dia\n",
"# 10 dias = 172800\n",
"# training df[0:172800] Normalize\n",
"# validation df[172800:224640] Don't normalize\n",
"# testing df[224640:276481] Don't normalize"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 82,
"metadata": {},
"outputs": [],
"source": [
"df = df[0:172800] "
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
"ruta_wandb_artifacts = 'data/PACMEL-2019/wandb_artifacts'# When permissions are fixed, save artifacts in 'data/PACMEL-2019/wandb_artifacts'\n",
"ar = TSArtifact.from_df(df, 'JNK', normalize=True, path= Path.home()/ruta_wandb_artifacts)"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'TS': {'sd': '2019-06-01 00:00:00',\n",
" 'ed': '2019-06-30 00:00:00',\n",
" 'ed': '2019-06-10 23:59:55',\n",
" 'created': 'from-df',\n",
" 'freq': '<5 * Seconds>',\n",
" 'n_vars': 16,\n",
" 'n_samples': 501121,\n",
" 'n_samples': 172800,\n",
" 'has_missing_values': 'False',\n",
" 'vars': ['RCD_AverageThree-phaseCurrent',\n",
" 'LCD_AverageThree-phaseCurrent',\n",
Expand All @@ -196,42 +227,42 @@
" 'SM_ShearerLocation',\n",
" 'SM_ShearerMoveInLeft',\n",
" 'SM_ShearerMoveInRight'],\n",
" 'normalization': {'means': {'RCD_AverageThree-phaseCurrent': 38.06796961212962,\n",
" 'LCD_AverageThree-phaseCurrent': 36.471058087767226,\n",
" 'LP_AverageThree-phaseCurrent': 1.9214672703798077,\n",
" 'LHD_LeftHaulageDrive(tractor)Temperature(gearbox)': 55.54040161956894,\n",
" 'RHD_RightHaulageDrive(tractor)Temperature(gearbox)': 2.8214171826764396,\n",
" 'LA_LeftArmTemperature': 41.23780524065046,\n",
" 'RA_RightArmTemperature': 44.15638338844311,\n",
" 'SM_DailyRouteOfTheShearer': 1104.077344393869,\n",
" 'SM_TotalRoute': 155.17678295262027,\n",
" 'LHD_EngineCurrent': 12.984789701489264,\n",
" 'RHD_EngineCurrent': 12.525651888466058,\n",
" 'RCD_BearingTemperature': 54.114771083231396,\n",
" 'SM_ShearerSpeed': 1.1442757338048097,\n",
" 'SM_ShearerLocation': 115.80804795648157,\n",
" 'SM_ShearerMoveInLeft': 0.07268523969260915,\n",
" 'SM_ShearerMoveInRight': 0.10187160386413663},\n",
" 'stds': {'RCD_AverageThree-phaseCurrent': 39.18209065397893,\n",
" 'LCD_AverageThree-phaseCurrent': 38.03455672086108,\n",
" 'LP_AverageThree-phaseCurrent': 2.688282374820025,\n",
" 'LHD_LeftHaulageDrive(tractor)Temperature(gearbox)': 22.16172387698709,\n",
" 'RHD_RightHaulageDrive(tractor)Temperature(gearbox)': 12.312526094478294,\n",
" 'LA_LeftArmTemperature': 11.447750876775066,\n",
" 'RA_RightArmTemperature': 13.085771183237727,\n",
" 'SM_DailyRouteOfTheShearer': 1036.3913516221676,\n",
" 'SM_TotalRoute': 25.04366490593873,\n",
" 'LHD_EngineCurrent': 21.93831294062176,\n",
" 'RHD_EngineCurrent': 21.21578937550054,\n",
" 'RCD_BearingTemperature': 15.336799266936271,\n",
" 'SM_ShearerSpeed': 2.9198292690278103,\n",
" 'SM_ShearerLocation': 78.26614419252932,\n",
" 'SM_ShearerMoveInLeft': 0.2554890106589118,\n",
" 'SM_ShearerMoveInRight': 0.2942006288396204}},\n",
" 'hash': '-2568758837685750672'}}"
" 'normalization': {'means': {'RCD_AverageThree-phaseCurrent': 37.00287326388889,\n",
" 'LCD_AverageThree-phaseCurrent': 35.23550983796297,\n",
" 'LP_AverageThree-phaseCurrent': 1.6554114583333333,\n",
" 'LHD_LeftHaulageDrive(tractor)Temperature(gearbox)': 55.24939930555555,\n",
" 'RHD_RightHaulageDrive(tractor)Temperature(gearbox)': 7.749042824074075,\n",
" 'LA_LeftArmTemperature': 38.294305555555546,\n",
" 'RA_RightArmTemperature': 45.11199421296297,\n",
" 'SM_DailyRouteOfTheShearer': 1050.3163686342593,\n",
" 'SM_TotalRoute': 129.42983975694443,\n",
" 'LHD_EngineCurrent': 12.796800347222224,\n",
" 'RHD_EngineCurrent': 12.471776041666669,\n",
" 'RCD_BearingTemperature': 55.206410300925924,\n",
" 'SM_ShearerSpeed': 1.1204710648148148,\n",
" 'SM_ShearerLocation': 120.40853993055558,\n",
" 'SM_ShearerMoveInLeft': 0.06525289351851851,\n",
" 'SM_ShearerMoveInRight': 0.10231539351851851},\n",
" 'stds': {'RCD_AverageThree-phaseCurrent': 37.83062195455937,\n",
" 'LCD_AverageThree-phaseCurrent': 34.32679724514462,\n",
" 'LP_AverageThree-phaseCurrent': 2.4762511407604766,\n",
" 'LHD_LeftHaulageDrive(tractor)Temperature(gearbox)': 25.85561071304018,\n",
" 'RHD_RightHaulageDrive(tractor)Temperature(gearbox)': 18.96694374934663,\n",
" 'LA_LeftArmTemperature': 12.625185345268926,\n",
" 'RA_RightArmTemperature': 16.54166491720409,\n",
" 'SM_DailyRouteOfTheShearer': 1069.2643430117464,\n",
" 'SM_TotalRoute': 11.225787346697574,\n",
" 'LHD_EngineCurrent': 22.343715753020742,\n",
" 'RHD_EngineCurrent': 21.791498792904196,\n",
" 'RCD_BearingTemperature': 15.94356411274981,\n",
" 'SM_ShearerSpeed': 2.9077431001835867,\n",
" 'SM_ShearerLocation': 89.69297001102812,\n",
" 'SM_ShearerMoveInLeft': 0.2438426450950956,\n",
" 'SM_ShearerMoveInRight': 0.29917885746919287}},\n",
" 'hash': '-2917046263098624813'}}"
]
},
"execution_count": null,
"execution_count": 84,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -242,37 +273,16 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 85,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[32m\u001b[41mERROR\u001b[0m Not authenticated. Copy a key from https://app.wandb.ai/authorize\n"
]
},
{
"name": "stdin",
"output_type": "stream",
"text": [
"API Key: ········································\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /home/user/.netrc\n"
]
},
{
"data": {
"text/html": [
"\n",
" Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
" Project page: <a href=\"https://app.wandb.ai/vrodriguezf/timecluster-extension\" target=\"_blank\">https://app.wandb.ai/vrodriguezf/timecluster-extension</a><br/>\n",
" Run page: <a href=\"https://app.wandb.ai/vrodriguezf/timecluster-extension/runs/tgcw9ai6\" target=\"_blank\">https://app.wandb.ai/vrodriguezf/timecluster-extension/runs/tgcw9ai6</a><br/>\n",
" Project page: <a href=\"https://app.wandb.ai/pacmel/timecluster-extension\" target=\"_blank\">https://app.wandb.ai/pacmel/timecluster-extension</a><br/>\n",
" Run page: <a href=\"https://app.wandb.ai/pacmel/timecluster-extension/runs/3dwqdbrr\" target=\"_blank\">https://app.wandb.ai/pacmel/timecluster-extension/runs/3dwqdbrr</a><br/>\n",
" "
],
"text/plain": [
Expand All @@ -286,22 +296,58 @@
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Wandb version 0.9.3 is available! To upgrade, please run:\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Wandb version 0.10.8 is available! To upgrade, please run:\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: $ pip install wandb --upgrade\n"
]
},
{
"data": {
"text/plain": [
"<timecluster_extension.load.TSArtifact at 0x7f08bfaa2940>"
]
},
"execution_count": 85,
"metadata": {},
"output_type": "execute_result"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Streaming file created twice in same run: /home/user/work/nbs/wandb/run-20201027_173944-3dwqdbrr/wandb-events.jsonl\n"
]
}
],
"source": [
"run = wandb.init(entity = 'pacmel', project='timecluster-extension', job_type='create_dataset', resume=True)\n",
"run.log_artifact(ar,aliases=['train_DR_2days'])"
"run.log_artifact(ar,aliases=['train_10days'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit fdf596a

Please sign in to comment.