site-add: attempt HTML to structured data

chapmanjacobd · Apr 17, 2024 · a55c035 · a55c035
1 parent c90153d
commit a55c035
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 128 deletions.
diff --git a/xklb/createdb/fs_add.py b/xklb/createdb/fs_add.py
@@ -317,134 +317,6 @@ def munge_image_tags(m: dict, e: dict) -> dict:
         ),
     }
 
-    for s in (
-        "Offset",
-        "Pattern",
-        "SubSampling",
-        "Zoom",
-        "Gain",
-        "FileNumber",
-        "RedEye",
-        "Conditional",
-        "RedBalance",
-        "BlueBalance",
-        "LensInfo",
-        "Shutter",
-        "FileSource",
-        "Chromaticities",
-        "Owner",
-        "DOF",
-        "Mode",
-        "Sensitivity",
-        "Exposure",
-        "Levels",
-        "Gamma",
-        "YCbCr",
-        "Serial",
-        "PDF",
-        "ObjectName",
-        "YCbCrPositioning",
-        ":ISO",
-        "ExposureProgram",
-        "FNumber",
-        "SensingMethod",
-        "SubjectDistance",
-        "Scene",
-        "Sharpness",
-        "Saturation",
-        "Flash",
-        "Contrast",
-        "MeteringMode",
-        "InteropIndex",
-        "ExposureCompensation",
-        "BrightnessValue",
-        "CustomRendered",
-        "ComponentsConfiguration",
-        "ShutterSpeed",
-        "ExposureMode",
-        "Aperture",
-        "ScaleFactor",
-        "LightValue",
-        "GPSProcessingMethod",
-        "GPSPosition",
-        "focalDistance",
-        "GPSDOP",
-        "FocalLength",
-        "FOV",
-        "CircleOfConfusion",
-        "GPSLatitudeRef",
-        "GPSLongitudeRef",
-        "Thumbnail",
-        "PrintStyle",
-        "Angle",
-        "Altitude",
-        "Displayed",
-        "WriterName",
-        "ReaderName",
-        "Date",
-        "History",
-        "Version",
-        "Compression",
-        "Digest",
-        "PrintPosition",
-        "PrintScale",
-        "Copyright",
-        "WhiteBalance",
-        "WhitePoint",
-        "ColorSpace",
-        "ColorTransform",
-        "ColorComponents",
-        "LightSource",
-        "Swatch",
-        "Profile",
-        "XMP:",
-        "ByteOrder",
-        "Comment",
-        "BitsPerSample",
-        "BitsPerPixel",
-        "Interpretation",
-        "EncodingProcess",
-        "Megapixels",
-        "PixelAspectRatio",
-        "ImageSize",
-        "PhotoshopFormat",
-        "OriginalTransmissionReference",
-        "Time",
-        "NumSlices",
-        "ImageUniqueID",
-        "HasRealMergedData",
-        "CodedCharacterSet",
-        "Flags0",
-        "Flags1",
-        "Padding",
-        "ProgressiveScans",
-        "HasColorMap",
-        "Ducky:Quality",
-        "PhotoshopQuality",
-        "SlicesGroupName",
-        "SupplementalCategories",
-        "Duration",
-        "Animation",
-        "FrameCount",
-    ):
-        pop_substring_keys(e, s)
-
-    for k in (
-        "File:FileName",
-        "File:Directory",
-        "File:FileSize",
-        "File:MIMEType",
-        "File:FilePermissions",
-        "File:FileTypeExtension",
-        "File:FileType",
-        "IPTC:SpecialInstructions",
-        "File:Exif",
-    ):
-        e.pop(k, None)
-
-    if e != {}:
-        log.info("Extra data %s", e)
-
     return m
 
 

diff --git a/xklb/createdb/site_add.py b/xklb/createdb/site_add.py
@@ -14,6 +14,7 @@ def parse_args() -> argparse.Namespace:
     parser.set_defaults(selenium=True)
 
     parser.add_argument("--local-file", "--local-html", action="store_true", help="Treat paths as Local HTML files")
+    parser.add_argument("--extract-html", action="store_true", help="Extract data from HTML")
 
     arggroups.debug(parser)
     arggroups.database(parser)
@@ -133,6 +134,29 @@ def response_interceptor(request, response):
             for d in tables:
                 db_thread[d["table_name"]].insert_all(iterables.list_dict_filter_bool(d["data"]), alter=True)  # type: ignore
 
+        if (
+            args.extract_html
+            and response
+            and response.status_code // 100 == 2  # HTTP 2xx
+            and "Content-Type" in response.headers
+            and response.headers["Content-Type"].startswith(("text/html",))
+        ):
+            body = decode(response.body, response.headers.get("Content-Encoding", "identity"))
+            body = body.decode()
+
+            import xmltodict
+
+            o = xmltodict.parse(body)
+            tables = nosql_to_sql(o)
+
+            if args.verbose > 2:
+                breakpoint()
+
+            tables = db_utils.add_missing_table_names(args, tables)
+            db_thread = db_utils.connect(argparse.Namespace(database=args.database, verbose=args.verbose))
+            for d in tables:
+                db_thread[d["table_name"]].insert_all(iterables.list_dict_filter_bool(d["data"]), alter=True)  # type: ignore
+
         elif (
             response
             and "Content-Type" in response.headers

diff --git a/xklb/usage.py b/xklb/usage.py
@@ -1673,6 +1673,11 @@ def play(action) -> str:
 
         library siteadd jobs.st.db --poke https://hk.jobsdb.com/hk/search-jobs/python/
 
+    Requires selenium-wire
+    Requires xmltodict when using --extract-html
+
+        pip install selenium-wire xmltodict
+
     Run with `-vv` to see and interact with the browser
 """