Merge pull request #288 from netarchivesuite/inline-image-filter

Attempt to filter out embedded images.
internetarchive · Jan 15, 2020 · 8e875bb · 8e875bb
2 parents 3a8b589 + 204491c
commit 8e875bb
Showing 1 changed file with 13 additions and 11 deletions.
diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
@@ -445,18 +445,20 @@ protected void processGeneralTag(CrawlURI curi, CharSequence element,
             } else if (attr.start(5) > -1) {
                 // SRC etc.
                 CharSequence context = elementContext(element, attr.group(5));
-
-                // true, if we expect another HTML page instead of an image etc.
-                final Hop hop;
-
-                if(!framesAsEmbeds
-                    && (elementStr.equalsIgnoreCase(FRAME) || elementStr
-                        .equalsIgnoreCase(IFRAME))) {
-                    hop = Hop.NAVLINK;
-                } else {
-                    hop = Hop.EMBED;
+                if (!context.toString().toLowerCase().startsWith("data:")) {
+
+                    // true, if we expect another HTML page instead of an image etc.
+                    final Hop hop;
+
+                    if (!framesAsEmbeds
+                            && (elementStr.equalsIgnoreCase(FRAME) || elementStr
+                            .equalsIgnoreCase(IFRAME))) {
+                        hop = Hop.NAVLINK;
+                    } else {
+                        hop = Hop.EMBED;
+                    }
+                    processEmbed(curi, value, context, hop);
                 }
-                processEmbed(curi, value, context, hop);
             } else if (attr.start(6) > -1) {
                 // CODEBASE
                 codebase = (value instanceof String)?