-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
165 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "markdown", | ||
| "id": "7858d66a-fb63-4aa0-a3bc-09a84af48ba4", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# Prepare seeds" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "f061eb7c-a615-44e0-bf6b-97215386df0a", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import os\n", | ||
| "import json\n", | ||
| "import gzip\n", | ||
| "import shutil\n", | ||
| "import requests\n", | ||
| "import zipfile\n", | ||
| "from pathlib import Path\n", | ||
| "from fhirpy import SyncFHIRClient" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "c097a082-37bd-487f-ac78-6a5b5fc5ae02", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "SOURCE = \"http://localhost:8080\"\n", | ||
| "client = SyncFHIRClient(SOURCE)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "20a511c3-5867-41ec-beff-3212c7881f44", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "def download_and_unzip(url, headers, extract_to):\n", | ||
| " response = requests.get(url, headers=headers)\n", | ||
| " if response.status_code == 200:\n", | ||
| " with open(\"temp.zip\", \"wb\") as f:\n", | ||
| " f.write(response.content)\n", | ||
| "\n", | ||
| " with zipfile.ZipFile(\"temp.zip\", 'r') as zip_ref:\n", | ||
| " zip_ref.extractall(extract_to)\n", | ||
| " os.remove(\"temp.zip\")\n", | ||
| " else:\n", | ||
| " print(\"Failed to download the file\")\n", | ||
| " return False\n", | ||
| " return True\n", | ||
| "\n", | ||
| "def filter_files(directory, exclude_strings):\n", | ||
| " for root, dirs, files in os.walk(directory):\n", | ||
| " for file in files:\n", | ||
| " if any(s in file for s in exclude_strings):\n", | ||
| " os.remove(os.path.join(root, file))\n", | ||
| "\n", | ||
| "def create_ndjson(input_dir, output_file):\n", | ||
| " data = []\n", | ||
| " for file_name in os.listdir(input_dir):\n", | ||
| " with open(os.path.join(input_dir, file_name), 'r') as file:\n", | ||
| " data.append(json.load(file))\n", | ||
| "\n", | ||
| " with open(output_file, 'w') as f:\n", | ||
| " for entry in data:\n", | ||
| " json.dump(entry, f)\n", | ||
| " f.write('\\n')\n", | ||
| "\n", | ||
| "def gzip_file(input_file, output_file):\n", | ||
| " with open(input_file, 'rb') as f_in:\n", | ||
| " with gzip.open(output_file, 'wb') as f_out:\n", | ||
| " shutil.copyfileobj(f_in, f_out)\n", | ||
| "\n", | ||
| "def move_file(src, dest):\n", | ||
| " shutil.move(src, dest)\n", | ||
| "\n", | ||
| "def clean_up(directory):\n", | ||
| " shutil.rmtree(directory)\n", | ||
| "\n", | ||
| "def to_aidbox_format(directory):\n", | ||
| " for filename in os.listdir(directory):\n", | ||
| " if filename.endswith('.json'):\n", | ||
| " file_path = os.path.join(directory, filename)\n", | ||
| " \n", | ||
| " with open(file_path, 'r') as file:\n", | ||
| " data = json.load(file)\n", | ||
| " result = client.execute(\"/$to-format/aidbox\", method='post', data=data, params=None) \n", | ||
| " with open(file_path, 'w') as file:\n", | ||
| " json.dump(result[\"resource\"], file, indent=4)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "8bfeee29-3754-4eac-954f-b366ac3e3d7b", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "url = \"https://hl7.org.au/fhir/core/0.2.2-preview/examples.json.zip\"\n", | ||
| "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}\n", | ||
| "include_strings = [\"StructureDefinition\", \"ValueSet\", \"CapabilityStatement\"]\n", | ||
| "extract_to = \"extracted\"\n", | ||
| "ndjson_file = \"seeds.ndjson\"\n", | ||
| "gzipped_file = \"seeds.ndjson.gz\"\n", | ||
| "destination_folder = \"../zenproject/\"" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "754cc3c2-8568-4ffd-9b6a-d37c2537b1c4", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "if download_and_unzip(url, headers, extract_to):\n", | ||
| " filter_files(extract_to, include_strings)\n", | ||
| " to_aidbox_format(extract_to)\n", | ||
| " create_ndjson(extract_to, ndjson_file)\n", | ||
| " gzip_file(ndjson_file, gzipped_file)\n", | ||
| " move_file(gzipped_file, Path(destination_folder) / gzipped_file)\n", | ||
| " clean_up(extract_to)\n", | ||
| " os.remove(ndjson_file)" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3 (ipykernel)", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.11.6" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 5 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.