From a248883c3580da0706ef9b4b59e446b99a25f557 Mon Sep 17 00:00:00 2001 From: Bastian Greshake Tzovaras Date: Fri, 20 Dec 2024 18:33:37 -0300 Subject: [PATCH 1/5] start support 4 download files into container --- Dockerfile | 2 + make-images.ipynb | 116 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4c46729..acbd5b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,4 +66,6 @@ RUN mkdir -p ${HOME}/output RUN mkdir -p ${HOME}/pgdata WORKDIR ${HOME} +RUN git clone https://github.com/geofabrik/sendfile_osm_oauth_protector + ENTRYPOINT ["./entrypoint-new.sh"] diff --git a/make-images.ipynb b/make-images.ipynb index 0e07465..0877fdb 100644 --- a/make-images.ipynb +++ b/make-images.ipynb @@ -23,11 +23,113 @@ "\n", "**The right file**: When you've picked your region, you will likely see two similarly named files listed: `regionname-internal.osh.pbf` and `regionname-latest-internal.osm.pbf`. The file you will want to download is the `regionname-internal.osh.pbf` (i.e. the one that includes `osh` and **no `latest`**).\n", "\n", - "Once you have downloaded your file onto your computer, you need to upload it into your container:\n", + "Once you have downloaded your file onto your computer, you need to upload it into your container. There are a few options that you can consider, depending on how you are using this notebook:\n", + "\n", + "#### Uploading a file you already have downloaded onto your computer\n", "\n", "* If you've launched this notebook in _MyBinder_, you can then open up the filebrowser on the top of the left hand side (The _file_ icon, or by pressing CTRL+SHIFT+F). \n", - "* If you've launched this notebook locally, you're likely in the _notebook_ interface, switch back to the _tree_ view and upload your file there (or put it into the shared folder which is accessible by both the container and your normal operating system). \n", + "* If you've launched this notebook locally, you're likely in the _notebook_ interface, switch back to the _tree_ view and upload your file there (or put it into the shared folder which is accessible by both the container and your normal operating system).\n", + "\n", + "#### Downloading a file directly from Geofabrik \n", + "\n", + "This option is particularly useful if you use this notebook on MyBinder, as it allows you to directly download the OSM history file into your Binder, without having to first download it onto your computer and then re-uploading it to the Binder, speeding things up considerably. As the OSM history files are only accessible with an OSM account, you will unfortunately have to fill out the form below, including your OSM username & password. \n", + "\n", + "**Note:** Your username & password will only be used to get a cookie that allows downloading the files and won't be permanently stored (and are not accessible to anyone else). If you are using _MyBinder_, your virtual machine will automatically shut down after more than 10 minutes of inactivity, destroying all data within the virtual machine in the process. If you are running this in a Docker container on your own computer the container will also destroy itself after shutting it down. \n", + "\n", + "If you decide to use the direct download from Geofabrik, enter the necessary details here: " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7d8cfd4c-913e-4983-92ef-c40567421331", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "03b1bf2d3a5447b3823c3b4007441b86", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Text(value='', description='OSM username:', placeholder='Steve'), Text(value='', description='O…" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ipywidgets as widgets\n", + "\n", + "osm_user = widgets.Text(\n", + " placeholder='Steve',\n", + " description='OSM username:',\n", + " disabled=False \n", + ")\n", + "\n", + "osm_password = widgets.Text(\n", + " placeholder='your password',\n", + " description='OSM password:',\n", + " disabled=False \n", + ")\n", "\n", + "file_url = widgets.Text(\n", + " placeholder='https://osm-internal.download.geofabrik.de/south-america/argentina-internal.osh.pbf',\n", + " description='Link to OSM history file:',\n", + " disabled=False \n", + ")\n", + "widgets.VBox([\n", + " osm_user,\n", + " osm_password,\n", + " file_url\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "15b753ea-4489-4620-9c19-e46e59f0afeb", + "metadata": {}, + "source": [ + "Once you have added the details above, you can download your file by running the cell below this one:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "582dbd2b-bd06-4dcf-934a-112c1de79fbf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-12-20 21:19:10-- https://osm-internal.download.geofabrik.de/south-america/uruguay-internal.osh.pbf\n", + "Resolving osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)... 65.109.22.172, 2a01:4f9:5a:1426::2\n", + "connected. to osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)|65.109.22.172|:443... \n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 87939811 (84M) [application/octet-stream]\n", + "Saving to: ‘uruguay-internal.osh.pbf’\n", + "\n", + "uruguay-internal.os 100%[===================>] 83.87M 190KB/s in 3m 3s \n", + "\n", + "2024-12-20 21:22:15 (468 KB/s) - ‘uruguay-internal.osh.pbf’ saved [87939811/87939811]\n", + "\n" + ] + } + ], + "source": [ + "!python sendfile_osm_oauth_protector/oauth_cookie_client.py -u \"{osm_user.value}\" -p \"{osm_password.value}\" -c \"https://osm-internal.download.geofabrik.de/get_cookie\" -f netscape -o geofabrik_cookie.txt;wget --max-redirect 0 --load-cookies geofabrik_cookie.txt {file_url.value}" + ] + }, + { + "cell_type": "markdown", + "id": "bfe0b5b8-cebd-4975-8447-f2189459dbcb", + "metadata": {}, + "source": [ + "Once the file is completely downloaded (or if you uploaded the file using one of the other methods), you can continue with the next step below. \n", "\n", "## Starting to make the comparison images. \n", "\n", @@ -38,22 +140,22 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 17, "id": "cf6cc023-ea39-46eb-885e-86a184dc1850", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "405e31685af24471838cd6587a52f032", + "model_id": "9637ee17e1b040d5857c2d36553015f8", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "VBox(children=(Dropdown(description='File:', options=('external/argentina-internal.osh.pbf', 'external/argenti…" + "VBox(children=(Dropdown(description='File:', options=('testfile.osh.pbf', 'uruguay-internal.osh.pbf', 'argenti…" ] }, - "execution_count": 3, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -63,7 +165,7 @@ "import glob\n", "\n", "filename = widgets.Dropdown(\n", - " options=glob.glob(\"*/*.pbf\"),\n", + " options=glob.glob(\"**/*.pbf\",recursive=True),\n", " description='File:',\n", " disabled=False,\n", ")\n", From 295ed70987a8e76b4b611870fc239a59bb950a22 Mon Sep 17 00:00:00 2001 From: Bastian Greshake Tzovaras Date: Sun, 22 Dec 2024 12:39:46 -0300 Subject: [PATCH 2/5] properly merge notebook --- make-images.ipynb | 180 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 155 insertions(+), 25 deletions(-) diff --git a/make-images.ipynb b/make-images.ipynb index 7d5b19e..06c756a 100644 --- a/make-images.ipynb +++ b/make-images.ipynb @@ -11,42 +11,169 @@ "\n", "## Getting started\n", "\n", - "### If this is your first time using a notebook like this:\n", + "### If this is your first time using a notebook like this: \n", "\n", "1. You can use the up/down arrow keys of your keyboard to move between the different \"cells\" or regions of this notebook. The currently selected one will be highlighed on the left with a colour bar.\n", "2. You can \"run\" each of these cells/regions by either pressing the \"play\" button in the menu above, or by pressing CTRL+ENTER on your keyboard.\n", - "3. If you are in the \"editing\" mode where you type into a cell, pressing ESC will take you back to the mode where the arrow keys work.\n", + "3. If you are in the \"editing\" mode where you type into a cell, pressing ESC will take you back to the mode where the arrow keys work. \n", "\n", "### Uploading the OSM history file\n", "\n", - "Before we can do the rest of the steps in this notebook, you will have to upload an OSM history file that covers your region of interest. You can find the corresponding files on [Geofabrik's internal download server](https://osm-internal.download.geofabrik.de/?landing_page=true). Login with your OSM account to get to the `internal` files, and then find the page for the your region of interest. Ideally, find the smallest region that covers your area of interest to minimize the filesize you need to upload.\n", + "Before we can do the rest of the steps in this notebook, you will have to upload an OSM history file that covers your region of interest. You can find the corresponding files on [Geofabrik's internal download server](https://osm-internal.download.geofabrik.de/?landing_page=true). Login with your OSM account to get to the `internal` files, and then find the page for the your region of interest. Ideally, find the smallest region that covers your area of interest to minimize the filesize you need to upload. \n", "\n", "**The right file**: When you've picked your region, you will likely see two similarly named files listed: `regionname-internal.osh.pbf` and `regionname-latest-internal.osm.pbf`. The file you will want to download is the `regionname-internal.osh.pbf` (i.e. the one that includes `osh` and **no `latest`**).\n", "\n", - "Once you have downloaded your file onto your computer, you need to upload it into your container:\n", + "Once you have downloaded your file onto your computer, you need to upload it into your container. There are a few options that you can consider, depending on how you are using this notebook:\n", "\n", - "* If you've launched this notebook in _MyBinder_, you can then open up the filebrowser on the top of the left hand side (The _file_ icon, or by pressing CTRL+SHIFT+F).\n", + "#### Uploading a file you already have downloaded onto your computer\n", + "\n", + "* If you've launched this notebook in _MyBinder_, you can then open up the filebrowser on the top of the left hand side (The _file_ icon, or by pressing CTRL+SHIFT+F). \n", "* If you've launched this notebook locally, you're likely in the _notebook_ interface, switch back to the _tree_ view and upload your file there (or put it into the shared folder which is accessible by both the container and your normal operating system).\n", "\n", + "#### Downloading a file directly from Geofabrik \n", "\n", - "## Starting to make the comparison images.\n", + "This option is particularly useful if you use this notebook on MyBinder, as it allows you to directly download the OSM history file into your Binder, without having to first download it onto your computer and then re-uploading it to the Binder, speeding things up considerably. As the OSM history files are only accessible with an OSM account, you will unfortunately have to fill out the form below, including your OSM username & password. \n", "\n", - "When you run the code-cell below, it will generate some \"forms\" that will feel familiar form other web-forms, which you can use to enter all the necessary parameters to create your images. These forms allow you to add the _history file_ you have just uploaded, the two time-points you want to compare, the bounding box (i.e. the area you're interested in, which you can find e.g. [via bboxfinder](https://bboxfinder.com)) and the minimum/maximum zoom levels, if you want to create images at different zoom levels.\n", + "**Note:** Your username & password will only be used to get a cookie that allows downloading the files and won't be permanently stored (and are not accessible to anyone else). If you are using _MyBinder_, your virtual machine will automatically shut down after more than 10 minutes of inactivity, destroying all data within the virtual machine in the process. If you are running this in a Docker container on your own computer the container will also destroy itself after shutting it down. \n", "\n", - "Now, run the cell below and then enter all the necessary information in the forms:" + "If you decide to use the direct download from Geofabrik, enter the necessary details here: " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7d8cfd4c-913e-4983-92ef-c40567421331", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0036ae1165254d019e392a3ebd9b7905", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Text(value='', description='OSM username:', placeholder='Steve'), Text(value='', description='O…" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ipywidgets as widgets\n", + "\n", + "osm_user = widgets.Text(\n", + " placeholder='Steve',\n", + " description='OSM username:',\n", + " disabled=False \n", + ")\n", + "\n", + "osm_password = widgets.Text(\n", + " placeholder='your password',\n", + " description='OSM password:',\n", + " disabled=False \n", + ")\n", + "\n", + "file_url = widgets.Text(\n", + " placeholder='https://osm-internal.download.geofabrik.de/south-america/argentina-internal.osh.pbf',\n", + " description='Link to OSM history file:',\n", + " disabled=False \n", + ")\n", + "widgets.VBox([\n", + " osm_user,\n", + " osm_password,\n", + " file_url\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "15b753ea-4489-4620-9c19-e46e59f0afeb", + "metadata": {}, + "source": [ + "Once you have added the details above, you can download your file by running the cell below this one:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "582dbd2b-bd06-4dcf-934a-112c1de79fbf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2024-12-20 21:19:10-- https://osm-internal.download.geofabrik.de/south-america/uruguay-internal.osh.pbf\n", + "Resolving osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)... 65.109.22.172, 2a01:4f9:5a:1426::2\n", + "connected. to osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)|65.109.22.172|:443... \n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 87939811 (84M) [application/octet-stream]\n", + "Saving to: ‘uruguay-internal.osh.pbf’\n", + "\n", + "uruguay-internal.os 100%[===================>] 83.87M 190KB/s in 3m 3s \n", + "\n", + "2024-12-20 21:22:15 (468 KB/s) - ‘uruguay-internal.osh.pbf’ saved [87939811/87939811]\n", + "\n" + ] + } + ], + "source": [ + "!python sendfile_osm_oauth_protector/oauth_cookie_client.py -u \"{osm_user.value}\" -p \"{osm_password.value}\" -c \"https://osm-internal.download.geofabrik.de/get_cookie\" -f netscape -o geofabrik_cookie.txt;wget --max-redirect 0 --load-cookies geofabrik_cookie.txt {file_url.value}" ] }, { + "cell_type": "markdown", + "id": "bfe0b5b8-cebd-4975-8447-f2189459dbcb", "metadata": {}, + "source": [ + "Once the file is completely downloaded (or if you uploaded the file using one of the other methods), you can continue with the next step below. \n", + "\n", + "## Starting to make the comparison images. \n", + "\n", + "When you run the code-cell below, it will generate some \"forms\" that will feel familiar form other web-forms, which you can use to enter all the necessary parameters to create your images. These forms allow you to add the _history file_ you have just uploaded, the two time-points you want to compare, the bounding box (i.e. the area you're interested in, which you can find e.g. [via bboxfinder](https://bboxfinder.com)) and the minimum/maximum zoom levels, if you want to create images at different zoom levels. \n", + "\n", + "Now, run the cell below and then enter all the necessary information in the forms:" + ] + }, + { "cell_type": "code", - "outputs": [], - "execution_count": null, + "execution_count": 1, + "id": "cf6cc023-ea39-46eb-885e-86a184dc1850", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ef5e2a5f18f54653bf1e7d39e90559ac", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Dropdown(description='File:', options=('external/argentina-internal.osh.pbf', 'external/argenti…" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import ipywidgets as widgets\n", "import glob\n", "\n", "filename = widgets.Dropdown(\n", - " options=glob.glob(\"*/*.pbf\"),\n", + " options=glob.glob(\"**/*.pbf\",recursive=True),\n", " description='File:',\n", " disabled=False,\n", ")\n", @@ -59,7 +186,7 @@ "bbox = widgets.Text(\n", " placeholder='-61.975228,-33.749067,-61.953167,-33.733014',\n", " description='BBox:',\n", - " disabled=False\n", + " disabled=False \n", ")\n", "\n", "\n", @@ -100,21 +227,19 @@ " date_label,\n", " min_zoom_level,\n", " max_zoom_level,\n", - " num_frames,\n", - "])" - ], - "id": "1b57c91aa6a007a6" + " num_frames])" + ] }, { "cell_type": "markdown", "id": "0fa54f42-e3f7-428b-b604-9450f4c659d7", "metadata": {}, "source": [ - "Once you have set your parameters, you can run the cell below in the same way that you ran the cell above for creating the forms.\n", + "Once you have set your parameters, you can run the cell below in the same way that you ran the cell above for creating the forms. \n", "\n", - "Running the cell below will take a while, in particular if it's the first time you execute it, as it needs to download some external resources first. Subsequent runs will be faster. You can re-run it as many times as you want, and change any of the parameters using the forms above.\n", + "Running the cell below will take a while, in particular if it's the first time you execute it, as it needs to download some external resources first. Subsequent runs will be faster. You can re-run it as many times as you want, and change any of the parameters using the forms above. \n", "\n", - "Once you run the cell below, it will print a lot of output into this notebook, including some warnings. If everything works, the last things printed out will read *Generating comparison image for zoom XXX*, according to the zoom levels you picked.\n", + "Once you run the cell below, it will print a lot of output into this notebook, including some warnings. If everything works, the last things printed out will read *Generating comparison image for zoom XXX*, according to the zoom levels you picked. \n", "\n", "The resulting images will be saved as \"progress.filename.*parameters*.png/gif." ] @@ -123,7 +248,9 @@ "cell_type": "code", "execution_count": 4, "id": "d5932adf-763e-4a3a-8f96-9247cb90ecc2", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -152,13 +279,13 @@ "}\n", "\n", "\n", - "\u001B[K\u001B[?25h \u001B[27m] | reify:lodash: \u001B[32;40mhttp\u001B[0m \u001B[35mfetch\u001B[0m GET 200 https://registry.npmjs.\u001B[0m\u001B[K.o\u001B[0m\u001B[K\n", + "\u001b[K\u001b[?25h \u001b[27m] | reify:lodash: \u001b[32;40mhttp\u001b[0m \u001b[35mfetch\u001b[0m GET 200 https://registry.npmjs.\u001b[0m\u001b[K.o\u001b[0m\u001b[K\n", "added 64 packages, and audited 65 packages in 5s\n", "\n", "1 package is looking for funding\n", " run `npm fund` for details\n", "\n", - "\u001B[31m\u001B[1m5\u001B[22m\u001B[39m vulnerabilities (2 \u001B[33m\u001B[1mmoderate\u001B[22m\u001B[39m, 3 \u001B[31m\u001B[1mhigh\u001B[22m\u001B[39m)\n", + "\u001b[31m\u001b[1m5\u001b[22m\u001b[39m vulnerabilities (2 \u001b[33m\u001b[1mmoderate\u001b[22m\u001b[39m, 3 \u001b[31m\u001b[1mhigh\u001b[22m\u001b[39m)\n", "\n", "To address all issues, run:\n", " npm audit fix\n", @@ -1554,7 +1681,9 @@ ] } ], - "source": "!./make.sh {filename.value} {start_date.value.isoformat()}Z {end_date.value.isoformat()}Z {bbox.value} {min_zoom_level.value} {max_zoom_level.value} {num_frames.value}" + "source": [ + "!./make.sh {filename.value} {start_date.value.isoformat()}Z {end_date.value.isoformat()}Z {bbox.value} {min_zoom_level.value} {max_zoom_level.value} {num_frames.value}" + ] }, { "cell_type": "markdown", @@ -1563,11 +1692,12 @@ "source": [ "Enjoy mapping and visualizing!\n", "\n", - "If this all worked you should be able to see the output files in the filebrowser/file tree, named `progress.filename.parameters.png/gif.`.\n", + "If this all worked you should be able to see the output files in the filebrowser/file tree, named `progress.filename.parameters.png/gif.`. \n", "\n", "If you are running this in the MyBinder, you should download them to your computer to keep them permanently.\n", "\n", - "If you are running this locally, you can do this as well or alternatively move them into the shared \"output\" folder that is accessible from your host computer." + "If you are running this locally, you can do this as well or alternatively move them into the shared \"output\" folder that is accessible from your host computer. \n", + "\n" ] } ], From 751d79b7911064142618d196a058042240f48bd4 Mon Sep 17 00:00:00 2001 From: Bastian Greshake Tzovaras Date: Sun, 22 Dec 2024 15:40:01 -0300 Subject: [PATCH 3/5] correct binder links after merge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5a7b7b6..2db1ec5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Make a image of OSM data of an area from 2 dates, showing what was changed. You can either install the necessary dependencies on your computer or run the script via Docker (both via MyBinder or in a local container). -[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/gedankenstuecke/osm-mapping-party-before-after/binderrize?labpath=make-images.ipynb) +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/amandasaurus/osm-mapping-party-before-after/main?labpath=make-images.ipynb) ## Making a local Installation @@ -34,7 +34,7 @@ You can either install the necessary dependencies on your computer or run the sc If you do not want to install the whole pipeline yourself, you can run this setup in a Docker container that is ready-to-use and that can also run in the free-to-use open source cloud infrastructure of _MyBinder_. -If you want to use the _MyBinder_ version, [click here and wait a bit](https://mybinder.org/v2/gh/gedankenstuecke/osm-mapping-party-before-after/binderrize?labpath=make-images.ipynb). This will launch the version online in a virtual machine and lets you interact with the code through a small Python notebook that will launch automatically and contains all necessary instructions. +If you want to use the _MyBinder_ version, [click here and wait a bit](https://mybinder.org/v2/gh/amandasaurus/osm-mapping-party-before-after/main?labpath=make-images.ipynb). This will launch the version online in a virtual machine and lets you interact with the code through a small Python notebook that will launch automatically and contains all necessary instructions. The MyBinder version has two drawbacks: 1. You will have to upload the OSM history file (_\*.osh.pbf_) into the container. Depending on the region of interest these can be quite large. 2. Creating the maps will take longer, as other external downloads will have to be downloaded on the fly. See [this blog post for more details](https://tzovar.as/map-comparisons/). From 25a094d75f87f44423735afab95ac7f74af1296e Mon Sep 17 00:00:00 2001 From: Bastian Greshake Tzovaras Date: Thu, 26 Dec 2024 11:57:45 -0300 Subject: [PATCH 4/5] add support for uploading cookie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Máté Gyöngyösi <13014176+gy-mate@users.noreply.github.com> --- make-images.ipynb | 154 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 22 deletions(-) diff --git a/make-images.ipynb b/make-images.ipynb index 06c756a..7143dca 100644 --- a/make-images.ipynb +++ b/make-images.ipynb @@ -17,7 +17,7 @@ "2. You can \"run\" each of these cells/regions by either pressing the \"play\" button in the menu above, or by pressing CTRL+ENTER on your keyboard.\n", "3. If you are in the \"editing\" mode where you type into a cell, pressing ESC will take you back to the mode where the arrow keys work. \n", "\n", - "### Uploading the OSM history file\n", + "## Getting an OSM history file\n", "\n", "Before we can do the rest of the steps in this notebook, you will have to upload an OSM history file that covers your region of interest. You can find the corresponding files on [Geofabrik's internal download server](https://osm-internal.download.geofabrik.de/?landing_page=true). Login with your OSM account to get to the `internal` files, and then find the page for the your region of interest. Ideally, find the smallest region that covers your area of interest to minimize the filesize you need to upload. \n", "\n", @@ -25,14 +25,20 @@ "\n", "Once you have downloaded your file onto your computer, you need to upload it into your container. There are a few options that you can consider, depending on how you are using this notebook:\n", "\n", - "#### Uploading a file you already have downloaded onto your computer\n", + "### 1. Uploading a file you already have downloaded onto your computer\n", "\n", "* If you've launched this notebook in _MyBinder_, you can then open up the filebrowser on the top of the left hand side (The _file_ icon, or by pressing CTRL+SHIFT+F). \n", "* If you've launched this notebook locally, you're likely in the _notebook_ interface, switch back to the _tree_ view and upload your file there (or put it into the shared folder which is accessible by both the container and your normal operating system).\n", "\n", - "#### Downloading a file directly from Geofabrik \n", + "### 2. Downloading a file directly from Geofabrik \n", "\n", - "This option is particularly useful if you use this notebook on MyBinder, as it allows you to directly download the OSM history file into your Binder, without having to first download it onto your computer and then re-uploading it to the Binder, speeding things up considerably. As the OSM history files are only accessible with an OSM account, you will unfortunately have to fill out the form below, including your OSM username & password. \n", + "This option is particularly useful if you use this notebook on MyBinder, as it allows you to directly download the OSM history file into your Binder, without having to first download it onto your computer and then re-uploading it to the Binder, speeding things up considerably. As the OSM history files are only accessible with an OSM account, you will have to be authenticated to use this method. \n", + "\n", + "There are two ways of achieving this to access files directly from _Geofabrik_. \n", + "\n", + "#### 2.1 Option a): Using your OSM username & password.\n", + "\n", + "You can use your OSM username & password to authenticate, by running the cell below and then entering the form fields. \n", "\n", "**Note:** Your username & password will only be used to get a cookie that allows downloading the files and won't be permanently stored (and are not accessible to anyone else). If you are using _MyBinder_, your virtual machine will automatically shut down after more than 10 minutes of inactivity, destroying all data within the virtual machine in the process. If you are running this in a Docker container on your own computer the container will also destroy itself after shutting it down. \n", "\n", @@ -41,18 +47,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "7d8cfd4c-913e-4983-92ef-c40567421331", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0036ae1165254d019e392a3ebd9b7905", + "model_id": "79eb82aff050415b9c87d9073a876a99", "version_major": 2, "version_minor": 0 }, @@ -60,7 +62,7 @@ "VBox(children=(Text(value='', description='OSM username:', placeholder='Steve'), Text(value='', description='O…" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -128,12 +130,124 @@ "!python sendfile_osm_oauth_protector/oauth_cookie_client.py -u \"{osm_user.value}\" -p \"{osm_password.value}\" -c \"https://osm-internal.download.geofabrik.de/get_cookie\" -f netscape -o geofabrik_cookie.txt;wget --max-redirect 0 --load-cookies geofabrik_cookie.txt {file_url.value}" ] }, + { + "cell_type": "markdown", + "id": "78358b23-cf0b-4890-83bc-70ead669b77d", + "metadata": {}, + "source": [ + "Once this is finished, you can move down to **Starting to make comparison images**!" + ] + }, + { + "cell_type": "markdown", + "id": "66cc684e-90e9-4ccb-a851-9315aaac5362", + "metadata": {}, + "source": [ + "#### 2.1 Option b): Using an existing Geofabrik cookie from your browser. \n", + "\n", + "This method allows you to paste the cookie that Geofabrik uses to check if you're allowed to download a file. As a result, **this method does not need to enter your OSM username/password**. The downside is that it's slightly more complicated, as it requires finding the cookie value in your web browser. \n", + "\n", + "To use this option, do the following: \n", + "\n", + "1. Head to the [OSM internal Geofabrik download page](https://osm-internal.download.geofabrik.de/)\n", + "2. Log-in on that page with your existing OSM credentials, this will create the necessary cookie. \n", + "3. Find the cookie in your web-browser: If you're using Firefox, press `F12` on your keyboard to open the web developer tools\n", + "4. Go to the tab called `Storage` in the bar that opens at the bottom of your browser\n", + "5. On the bottom-left, you will see `Cookies`, click on it and select the cookie that starts with `https://osm-internal.download.geofabrik.de`.\n", + "6. You will see a Name/Value pair displayed, with the name reading `gf_download_oauth`. Select this cookie\n", + "7. On the right-hand side you will see a `Data` option, which gives the full cookie contents. You can right-click on this and select `Copy`.\n", + "\n", + "You can then paste this cookie data into the form below, alongside the link to the file you want to download:" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "e5f626cc-f267-4f19-a290-17144f1ffde9", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1f012554f3fb458b84fa56c27973df37", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Text(value='', description='gf_download_oauth cookie value:', placeholder='login|2018-04-12|WVR…" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "geofabrik_cookie = ('gf_download_oauth', widgets.Text(\n", + " placeholder='login|2018-04-12|WVRced34c...',\n", + " description='gf_download_oauth cookie value:',\n", + " disabled=False))\n", + "file_url = widgets.Text(\n", + " placeholder='https://osm-internal.download.geofabrik.de/south-america/argentina-internal.osh.pbf',\n", + " description='Link to OSM history file:',\n", + " disabled=False)\n", + "\n", + "widgets.VBox([\n", + " geofabrik_cookie[1],\n", + " file_url])" + ] + }, + { + "cell_type": "markdown", + "id": "55e67302-5653-4f4f-b2c5-3963379e4ecc", + "metadata": {}, + "source": [ + "Once you have added those, run the cell below to download your file." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "4eb7ed00-1902-4d5e-8ba3-f149ad85c61d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "login|2018-04-12|eRRYaHdVQwFceNZlGOUawLbKH8oR-OjHNwBmVcnkpOaJJa0-H9X6zfFgJLWrfwbGXzdyh7bNUPH1WjrvkkpWAY4kI6rSSg_A_HP2CSpAS4XEr91hLamrnPsrgbYETIUWkvKRrFzVthHkHJDALGVmd7E8fgNKe_uL9xeJaKAWU_Ydi_uu4a86xKhgkfZv1NBI_g27phWwTtvt-TE57v9Vygc_jgozXpnmL3ENny3g\n", + "--2024-12-26 14:53:24-- https://osm-internal.download.geofabrik.de/europe/malta-internal.osh.pbf\n", + "Resolving osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)... 65.109.22.172, 2a01:4f9:5a:1426::2\n", + "connected. to osm-internal.download.geofabrik.de (osm-internal.download.geofabrik.de)|65.109.22.172|:443... \n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 18084230 (17M) [application/octet-stream]\n", + "Saving to: ‘malta-internal.osh.pbf’\n", + "\n", + "malta-internal.osh. 100%[===================>] 17.25M 4.12MB/s in 4.2s \n", + "\n", + "2024-12-26 14:53:29 (4.12 MB/s) - ‘malta-internal.osh.pbf’ saved [18084230/18084230]\n", + "\n" + ] + } + ], + "source": [ + "cookie_name = geofabrik_cookie[0]\n", + "if geofabrik_cookie[1].value.startswith(\"gf_download_oauth\"):\n", + " cookie_value = geofabrik_cookie[1].value.replace('gf_download_oauth:\"','')\n", + " cookie_value = cookie_value[:-1]\n", + " print(cookie_value)\n", + "else:\n", + " cookie_value = geofabrik_cookie[1].value\n", + " \n", + "!wget --max-redirect 0 --header=\"Cookie: {cookie_name}={cookie_value}\" {file_url.value}" + ] + }, { "cell_type": "markdown", "id": "bfe0b5b8-cebd-4975-8447-f2189459dbcb", "metadata": {}, "source": [ - "Once the file is completely downloaded (or if you uploaded the file using one of the other methods), you can continue with the next step below. \n", + "Once the file is completely downloaded (or if you uploaded the file using one of the other methods above), you can continue with the next step below. \n", "\n", "## Starting to make the comparison images. \n", "\n", @@ -144,26 +258,22 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, "id": "cf6cc023-ea39-46eb-885e-86a184dc1850", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef5e2a5f18f54653bf1e7d39e90559ac", + "model_id": "2bd9d49b689145f093b8fe9445d0ac7c", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "VBox(children=(Dropdown(description='File:', options=('external/argentina-internal.osh.pbf', 'external/argenti…" + "VBox(children=(Dropdown(description='File:', options=('uruguay-internal.osh.pbf', 'external/argentina-internal…" ] }, - "execution_count": 1, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } From 47417ef4a0aa52c2c1fe37fa455840a577cf5871 Mon Sep 17 00:00:00 2001 From: Bastian Greshake Tzovaras Date: Fri, 27 Dec 2024 19:18:00 -0300 Subject: [PATCH 5/5] Update make-images.ipynb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Máté Gyöngyösi --- make-images.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/make-images.ipynb b/make-images.ipynb index 7143dca..825d995 100644 --- a/make-images.ipynb +++ b/make-images.ipynb @@ -150,10 +150,10 @@ "To use this option, do the following: \n", "\n", "1. Head to the [OSM internal Geofabrik download page](https://osm-internal.download.geofabrik.de/)\n", - "2. Log-in on that page with your existing OSM credentials, this will create the necessary cookie. \n", - "3. Find the cookie in your web-browser: If you're using Firefox, press `F12` on your keyboard to open the web developer tools\n", + "2. Log in on that page with your existing OSM credentials, this will create the necessary cookie. \n", + "3. Find the cookie in your web browser: If you're using Firefox, press `F12` on your keyboard to open the web developer tools\n", "4. Go to the tab called `Storage` in the bar that opens at the bottom of your browser\n", - "5. On the bottom-left, you will see `Cookies`, click on it and select the cookie that starts with `https://osm-internal.download.geofabrik.de`.\n", + "5. On the bottom left, you will see `Cookies`, click on it and select the cookie that starts with `https://osm-internal.download.geofabrik.de`.\n", "6. You will see a Name/Value pair displayed, with the name reading `gf_download_oauth`. Select this cookie\n", "7. On the right-hand side you will see a `Data` option, which gives the full cookie contents. You can right-click on this and select `Copy`.\n", "\n",