Rename effectlive tld to public suffic, move before classification, a…

…dd same primary domain as a classification for internal domains

src/aggregate/analysis.sh

@@ -130,6 +130,7 @@ def mangleShared:
 				"is-same-domain": .classification.isSameDomain,
 				"is-subdomain": .classification.isSubdomain,
 				"is-superdomain": .classification.isSuperdomain,
+				"is-same-primary-domain": .classification.isSamePrimaryDomain,
 				"is-internal-domain": .classification.isInternalDomain,
 				"is-external-domain": .classification.isExternalDomain,
 				"is-successful-request": .classification.isSuccessful,

src/aggregate/prepare.sh

@@ -96,7 +96,7 @@ def mangleUrl:
 			.domain | {
 					value: .value,
 					"public-suffixes": (if ."public-suffixes" then (."public-suffixes" | map(.idn | fallbackString)) else null end),
-					"primary-domain": (if ."private-prefixes" then (."private-prefixes"[-1:][0] | fallbackString) else null end),
+					"primary-domain",
 			}
 		)
 	};
@@ -126,7 +126,7 @@ def distinctMangleDomain(domain):
 	domain as $domain
 	| .value |= setKeyCounterObjectCount($domain.value | fallbackString; 1)
 	| ."public-suffixes" |= setArrayToKeyCounterObject(($domain."public-suffixes" // []) | map(.idn | fallbackString); 1)
-	| ."primary-domain" |= setKeyCounterObjectCount(($domain."private-prefixes" // []) | .[-1:][0] | fallbackString; 1);
+	| ."primary-domain" |= setKeyCounterObjectCount($domain."primary-domain" | fallbackString; 1);
 
 def distinctMangleUrl(url):
 	. as $aggregatedUrl
@@ -166,6 +166,7 @@ def distinctMangle:
 				isSameDomain: (.[0].classification.isSameDomain // false),
 				isSubdomain: (.[0].classification.isSubdomain // false),
 				isSuperdomain: (.[0].classification.isSuperdomain // false),
+				isSamePrimaryDomain: (.[0].classification.isSamePrimaryDomain // false),
 				isInternalDomain: (.[0].classification.isInternalDomain // false),
 				isExternalDomain: (.[0].classification.isExternalDomain // false),
 				isSuccessful: (.[0].classification.isSuccessful // false),
@@ -196,6 +197,7 @@ def distinctMangle:
 		.classification.isSameDomain = (.classification.isSameDomain and $request.classification.isSameDomain)
 		| .classification.isSubdomain = (.classification.isSubdomain and $request.classification.isSubdomain)
 		| .classification.isSuperdomain = (.classification.isSuperdomain and $request.classification.isSuperdomain)
+		| .classification.isSamePrimaryDomain = (.classification.isSamePrimaryDomain and $request.classification.isSamePrimaryDomain)
 		| .classification.isInternalDomain = (.classification.isInternalDomain and $request.classification.isInternalDomain)
 		| .classification.isExternalDomain = (.classification.isExternalDomain and $request.classification.isExternalDomain)
 		| .classification.isSuccessful = (.classification.isSuccessful and $request.classification.isSuccessful)

src/aggregate/prepare2.sh

@@ -133,6 +133,7 @@ def base:
 			isSameDomain: 0,
 			isSubdomain: 0,
 			isSuperdomain: 0,
+			isSamePrimaryDomain: 0,
 			isInternalDomain: 0,
 			isExternalDomain: 0,
 			isSuccessful: 0,
@@ -177,6 +178,7 @@ def mangleClassification(request):
 	| .classification.isSameDomain += ($request.classification.isSameDomain | boolToInt)
 	| .classification.isSubdomain += ($request.classification.isSubdomain | boolToInt)
 	| .classification.isSuperdomain += ($request.classification.isSuperdomain | boolToInt)
+	| .classification.isSamePrimaryDomain += ($request.classification.isSamePrimaryDomain | boolToInt)
 	| .classification.isInternalDomain += ($request.classification.isInternalDomain | boolToInt)
 	| .classification.isExternalDomain += ($request.classification.isExternalDomain | boolToInt)
 	| .classification.isSuccessful += ($request.classification.isSuccessful | boolToInt)
@@ -235,6 +237,7 @@ def distinctBase:
 			isSameDomain: 0,
 			isSubdomain: 0,
 			isSuperdomain: 0,
+			isSamePrimaryDomain: 0,
 			isInternalDomain: 0,
 			isExternalDomain: 0,
 			isSuccessful: 0,
@@ -278,7 +281,7 @@ def distinctMangleClassification(request):
 	request as $request
 	| .classification.isSameDomain += ($request.classification.isSameDomain | boolToInt)
 	| .classification.isSubdomain += ($request.classification.isSubdomain | boolToInt)
-	| .classification.isSuperdomain += ($request.classification.isSuperdomain | boolToInt)
+	| .classification.isSamePrimaryDomain += ($request.classification.isSamePrimaryDomain | boolToInt)
 	| .classification.isInternalDomain += ($request.classification.isInternalDomain | boolToInt)
 	| .classification.isExternalDomain += ($request.classification.isExternalDomain | boolToInt)
 	| .classification.isSuccessful += ($request.classification.isSuccessful | boolToInt)

src/classification/basic.sh

@@ -18,6 +18,10 @@ def isSuperdomain(domain):
 	| $domain
 	| isSubdomain($original);
 
+def isSamePrimaryDomain(originDomain):
+	originDomain as $originDomain
+	| ."primary-domain" == $originDomain."primary-domain";
+
 def isSecure:
 	. == "https";
 
@@ -28,12 +32,14 @@ def classifyUrl(origin):
 	| (if $hasDomainValue then (.domain.value | isSameDomain($origin.domain.value)) else false end) as $isSameDomain
 	| (if $hasDomainValue then (.domain.value | isSubdomain($origin.domain.value)) else false end) as $isSubdomain
 	| (if $hasDomainValue then (.domain.value | isSuperdomain($origin.domain.value)) else false end) as $isSuperdomain
-	| ($isSameDomain or $isSubdomain or $isSuperdomain) as $isInternalDomain
+	| (if $hasDomainValue then (.domain | isSamePrimaryDomain($origin.domain)) else false end) as $isSamePrimaryDomain
+	| ($isSameDomain or $isSubdomain or $isSuperdomain or $isSamePrimaryDomain) as $isInternalDomain
 	| (if (.scheme and .scheme.valid and .scheme.value) then (.scheme.value | isSecure) else false end) as $isSecure
 	| {
 		isSameDomain: $isSameDomain,
 		isSubdomain: $isSubdomain,
 		isSuperdomain: $isSuperdomain,
+		isSamePrimaryDomain: $isSamePrimaryDomain,
 		isInternalDomain: $isInternalDomain,
 		isExternalDomain: ($isInternalDomain | not),
 		isSecure: $isSecure,

src/classification/disconnect/prepare-service-list.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-read -d '' classifyExpandedParts <<-'EOF' || true
+read -d '' prepareServiceList <<-'EOF' || true
 def toNullOrSingleValueOrArray:
 	if length == 0 then
 		# Replace an empty array with null.
@@ -71,4 +71,4 @@ def transformRawDisconnect:
 | transformRawDisconnect
 EOF
 
-cat | jq "$classifyExpandedParts"
+cat | jq "$prepareServiceList"

src/classification/effective-tld/add.sh → src/classification/public-suffix/add.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
-effectiveTldFile="$1"
+preparedPublicSuffixFile="$1"
 
-read -d '' classifyExpandedParts <<-'EOF' || true
+read -d '' addPublicSuffixes <<-'EOF' || true
 def deleteNullKey(key):
 	# Delete a property if it is null.
 	key as $key
@@ -32,13 +32,13 @@ def deleteEmptyArrayKey(key):
 		)
 	);
 
-def matchEffectiveTld:
-	# Match the domain to all possible rules/groups/public-suffixes in the effective tld list.
+def matchPublicSuffix:
+	# Match the domain to all possible rules/groups/public-suffixes in the public suffix list.
 	map(
-		# has($subdomain) is more effective than $effectiveTld[.] // empty
+		# has($subdomain) is more effective than $publicSuffixLookup[.] // empty
 		. as $subdomain
-		| if $effectiveTld | has($subdomain) then
-			$effectiveTld[$subdomain]
+		| if $publicSuffixLookup | has($subdomain) then
+			$publicSuffixLookup[$subdomain]
 		else
 			empty
 		end
@@ -49,11 +49,16 @@ def getPrivatePrefix(publicSuffixes):
 	| ($publicSuffixes | length) as $publicSuffixesLength
 	| .[0:(length - $publicSuffixesLength)];
 
+def getPrimaryDomain:
+	.[-1:][0];
+
 def mangle:
 	if . and .domain and .domain.components and (.domain.components | type) == "array" and (.domain.components | length) > 0 then
-		(.domain.components | matchEffectiveTld) as $currentSuffixes
+		(.domain.components | matchPublicSuffix) as $currentSuffixes
+		| (.domain.components | getPrivatePrefix($currentSuffixes)) as $currentPrefixes
 		| .domain."public-suffixes" = $currentSuffixes
-		| .domain."private-prefixes" = (.domain.components | getPrivatePrefix($currentSuffixes))
+		| .domain."private-prefixes" = $currentPrefixes
+		| .domain."primary-domain" = ($currentPrefixes | getPrimaryDomain)
 	else
 		.
 	end;
@@ -63,4 +68,4 @@ def mangle:
 | .requestedUrls[].referer |= mangle
 EOF
 
-cat | jq "$classifyExpandedParts" --argfile "effectiveTld" "$effectiveTldFile"
+cat | jq "$addPublicSuffixes" --argfile "publicSuffixLookup" "$preparedPublicSuffixFile"

src/multiset/non-failed.classification.domain-scope.coverage.sh

@@ -10,6 +10,7 @@ read -d '' getNonFailedClassificationDomainScope <<-'EOF' || true
 	"is-same-domain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-same-domain",
 	"is-subdomain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-subdomain",
 	"is-superdomain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-superdomain",
+	"is-same-primary-domain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-same-primary-domain",
 	"is-internal-domain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-internal-domain",
 	"is-external-domain": .successfulOrigin.unfilteredUrls.requestedUrlsDistinct.coverage.classification."is-external-domain",
 }
@@ -23,6 +24,7 @@ read -d '' mapData <<-'EOF' || true
 	"is-same-domain",
 	"is-subdomain",
 	"is-superdomain",
+	"is-same-primary-domain",
 	"is-internal-domain",
 	"is-external-domain",
 	"is-mixed-domain": (1 - ."is-internal-domain" - ."is-external-domain"),
@@ -37,9 +39,10 @@ map(
 		"03--Same domain": ."is-same-domain",
 		"04--Subdomain": ."is-subdomain",
 		"05--Superdomain": ."is-superdomain",
-		"06--Internal domain": ."is-internal-domain",
-		"07--External domain": ."is-external-domain",
-		"08--Mixed": ."is-mixed-domain",
+		"06--Same primary": ."is-same-primary-domain",
+		"07--Internal domain": ."is-internal-domain",
+		"08--External domain": ."is-external-domain",
+		"09--Mixed": ."is-mixed-domain",
 	}
 )
 EOF

src/one-shot/aggregate.sh

@@ -2,7 +2,7 @@
 set -e
 
 # Prepare aggregates base
-<"domains.parts.expanded.classified.disconnect.effective-tld.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../aggregate/prepare.sh" > "aggregates.base.json"
+<"domains.parts.expanded.public-suffix.classified.disconnect.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../aggregate/prepare.sh" > "aggregates.base.json"
 
 # Aggregates
 <"aggregates.base.json" "${BASH_SOURCE%/*}/../aggregate/all.sh" > "aggregates.json"

src/one-shot/data.sh

@@ -20,11 +20,11 @@ domainroot=$(cd -- "$domainroot"; echo "$PWD")
 # Expand parts by splitting them up to parts
 <"domains.parts.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../extract/request/expand-parts.sh" > "domains.parts.expanded.json"
 
-# Add basic classifications
-<"domains.parts.expanded.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/basic.sh" > "domains.parts.expanded.classified.json"
+# Add public suffix domain grouping
+<"domains.parts.expanded.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/public-suffix/add.sh" "prepared.public-suffix.json" > "domains.parts.expanded.public-suffix.json"
 
-# Add disconnect's block matching
-<"domains.parts.expanded.classified.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/disconnect/add.sh" "prepared.disconnect.services.json" > "domains.parts.expanded.classified.disconnect.json"
+# Add basic classifications
+<"domains.parts.expanded.public-suffix.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/basic.sh" > "domains.parts.expanded.public-suffix.classified.json"
 
-# Add effective tld domain grouping
-<"domains.parts.expanded.classified.disconnect.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/effective-tld/add.sh" "prepared.effective-tld.json" > "domains.parts.expanded.classified.disconnect.effective-tld.json"
+# # Add disconnect's block matching
+<"domains.parts.expanded.public-suffix.classified.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../classification/disconnect/add.sh" "prepared.disconnect.services.json" > "domains.parts.expanded.public-suffix.classified.disconnect.json"

src/one-shot/preparations.sh

@@ -17,5 +17,5 @@ download "https://services.disconnect.me/disconnect-plaintext.json"
 
 
 download "https://publicsuffix.org/list/effective_tld_names.dat"
-<"effective_tld_names.dat" "${BASH_SOURCE%/*}/../classification/effective-tld/prepare-list.sh" > "prepared.effective-tld.json"
+<"effective_tld_names.dat" "${BASH_SOURCE%/*}/../classification/public-suffix/prepare-list.sh" > "prepared.public-suffix.json"
 

src/one-shot/questions.sh

@@ -7,7 +7,7 @@ set -e
 <"google-gtm-ga-dc.json" "${BASH_SOURCE%/*}/../questions/google-gtm-ga-dc.aggregate.sh" > "google-gtm-ga-dc.aggregate.json"
 
 # Origin redirects
-<"domains.parts.expanded.classified.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../questions/origin-redirects.sh" > "origin-redirects.json"
+<"domains.parts.expanded.public-suffix.classified.json" "${BASH_SOURCE%/*}/../util/parallel-chunks.sh" "${BASH_SOURCE%/*}/../questions/origin-redirects.sh" > "origin-redirects.json"
 # TODO: parallelize question aggregation?
 <"origin-redirects.json" "${BASH_SOURCE%/*}/../questions/origin-redirects.aggregate.sh" > "origin-redirects.aggregate.json"
 

src/questions/origin-redirects.aggregate.sh

@@ -30,6 +30,7 @@ def countsBase:
 		isSameDomain: 0,
 		isSubdomain: 0,
 		isSuperdomain: 0,
+		isSamePrimaryDomain: 0,
 		isInternalDomain: 0,
 		isExternalDomain: 0,
 		isSecure: 0,

src/questions/origin-redirects.sh

@@ -174,6 +174,7 @@ select(
 	isSameDomain: 0,
 	isSubdomain: 0,
 	isSuperdomain: 0,
+	isSamePrimaryDomain: 0,
 	isInternalDomain: 0,
 	isExternalDomain: 0,
 	isSecure: 0,
@@ -183,6 +184,7 @@ select(
 | .counts.isSameDomain = redirectClassificationCount(.isSameDomain)
 | .counts.isSubdomain = redirectClassificationCount(.isSubdomain)
 | .counts.isSuperdomain = redirectClassificationCount(.isSuperdomain)
+| .counts.isSamePrimaryDomain = redirectClassificationCount(.isSamePrimaryDomain)
 | .counts.isInternalDomain = redirectClassificationCount(.isInternalDomain)
 | .counts.isExternalDomain = redirectClassificationCount(.isExternalDomain)
 | .counts.isSecure = redirectClassificationCount(.isSecure)