Skip to content

[Smart hashing] Cleanup #2875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 0 additions & 51 deletions packages/core/src/__tests__/hashing-utils.test.ts

This file was deleted.

40 changes: 0 additions & 40 deletions packages/core/src/hashing-utils.ts

This file was deleted.

1 change: 0 additions & 1 deletion packages/core/src/index.ts
Original file line number Diff line number Diff line change
@@ -48,7 +48,6 @@ export { retry } from './retry'
export { get } from './get'
export { omit } from './omit'
export { removeUndefined } from './remove-undefined'
export { sha256SmartHash, SmartHashing, sha1Hash } from './hashing-utils'
export { time, duration } from './time'

export { realTypeOf, isObject, isArray, isString } from './real-type-of'
100 changes: 18 additions & 82 deletions packages/destination-actions/src/destinations/amazon-amc/function.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Features, InvalidAuthenticationError } from '@segment/actions-core'
import { InvalidAuthenticationError } from '@segment/actions-core'
import { JSONLikeObject, MultiStatusResponse, PayloadValidationError, RequestClient } from '@segment/actions-core'
import { AudienceSettings, Settings } from './generated-types'
import type { Payload } from './syncAudiencesToDSP/generated-types'
@@ -10,10 +10,9 @@ export async function processPayload(
request: RequestClient,
settings: Settings,
payload: Payload[],
audienceSettings: AudienceSettings,
features?: Features
audienceSettings: AudienceSettings
) {
const payloadRecord = createPayloadToUploadRecords(payload, audienceSettings, features)
const payloadRecord = createPayloadToUploadRecords(payload, audienceSettings)
// Regular expression to find a audienceId numeric string and replace the quoted audienceId string with an unquoted number
const payloadString = JSON.stringify(payloadRecord).replace(/"audienceId":"(\d+)"/, '"audienceId":$1')

@@ -46,19 +45,15 @@ export async function processPayload(
* @throws {PayloadValidationError} - Throws an error if any externalUserId does not
* match the expected pattern.
*/
export function createPayloadToUploadRecords(
payloads: Payload[],
audienceSettings: AudienceSettings,
features?: Features
) {
export function createPayloadToUploadRecords(payloads: Payload[], audienceSettings: AudienceSettings) {
const records: AudienceRecord[] = []
const { audienceId } = payloads[0]
payloads.forEach((payload: Payload) => {
// Check if the externalUserId matches the pattern
if (!REGEX_EXTERNALUSERID.test(payload.externalUserId)) {
return // Skip to the next iteration
}
const hashedPII = hashedPayload(payload, features)
const hashedPII = hashedPayload(payload)
const payloadRecord: AudienceRecord = {
externalUserId: payload.externalUserId,
countryCode: audienceSettings.countryCode,
@@ -83,8 +78,7 @@ export function createPayloadToUploadRecords(
function validateAndPreparePayload(
payloads: Payload[],
multiStatusResponse: MultiStatusResponse,
audienceSettings: AudienceSettings,
features?: Features
audienceSettings: AudienceSettings
) {
const validPayloadIndicesBitmap: number[] = []
const filteredPayloads: AudienceRecord[] = []
@@ -99,7 +93,7 @@ function validateAndPreparePayload(
return
}

const hashedPII = hashedPayload(payload, features)
const hashedPII = hashedPayload(payload)
const payloadRecord: AudienceRecord = {
externalUserId: payload.externalUserId,
countryCode: audienceSettings.countryCode,
@@ -130,15 +124,13 @@ export async function processBatchPayload(
request: RequestClient,
settings: Settings,
payloads: Payload[],
audienceSettings: AudienceSettings,
features?: Features
audienceSettings: AudienceSettings
) {
const multiStatusResponse = new MultiStatusResponse()
const { filteredPayloads, validPayloadIndicesBitmap } = validateAndPreparePayload(
payloads,
multiStatusResponse,
audienceSettings,
features
audienceSettings
)

if (!filteredPayloads.length) {
@@ -237,88 +229,32 @@ function normalizeEmail(email: string): string {
* @returns {HashedPIIObject} - The object containing the hashed PII fields.
*/

function hashedPayload(payload: Payload, features?: Features): HashedPIIObject {
function hashedPayload(payload: Payload): HashedPIIObject {
const hashedPII: HashedPIIObject = {}

if (payload.firstName) {
hashedPII.firstname = processHashing(
payload.firstName,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.firstname = processHashing(payload.firstName, 'sha256', 'hex', normalizeStandard)
}
if (payload.lastName) {
hashedPII.lastname = processHashing(
payload.lastName,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.lastname = processHashing(payload.lastName, 'sha256', 'hex', normalizeStandard)
}
if (payload.address) {
hashedPII.address = processHashing(
payload.address,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.address = processHashing(payload.address, 'sha256', 'hex', normalizeStandard)
}
if (payload.postal) {
hashedPII.postal = processHashing(
payload.postal,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.postal = processHashing(payload.postal, 'sha256', 'hex', normalizeStandard)
}
if (payload.phone) {
hashedPII.phone = processHashing(
payload.phone,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizePhone
)
hashedPII.phone = processHashing(payload.phone, 'sha256', 'hex', normalizePhone)
}
if (payload.city) {
hashedPII.city = processHashing(
payload.city,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.city = processHashing(payload.city, 'sha256', 'hex', normalizeStandard)
}
if (payload.state) {
hashedPII.state = processHashing(
payload.state,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeStandard
)
hashedPII.state = processHashing(payload.state, 'sha256', 'hex', normalizeStandard)
}
if (payload.email) {
hashedPII.email = processHashing(
payload.email,
'sha256',
'hex',
features ?? {},
'actions-amazon-amc',
normalizeEmail
)
hashedPII.email = processHashing(payload.email, 'sha256', 'hex', normalizeEmail)
}

return hashedPII
Original file line number Diff line number Diff line change
@@ -190,8 +190,7 @@ describe('AmazonAds.syncAudiencesToDSP', () => {
}
},
settings,
useDefaultMappings: true,
features: { 'smart-hashing': true }
useDefaultMappings: true
})

expect(response.length).toBe(1)
Original file line number Diff line number Diff line change
@@ -121,11 +121,11 @@ const action: ActionDefinition<Settings, Payload> = {
default: 10000
}
},
perform: (request, { settings, payload, audienceSettings, features }) => {
return processPayload(request, settings, [payload], audienceSettings, features)
perform: (request, { settings, payload, audienceSettings }) => {
return processPayload(request, settings, [payload], audienceSettings)
},
performBatch: async (request, { settings, payload: payloads, audienceSettings, features }) => {
return await processBatchPayload(request, settings, payloads, audienceSettings, features)
performBatch: async (request, { settings, payload: payloads, audienceSettings }) => {
return await processBatchPayload(request, settings, payloads, audienceSettings)
}
}

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ActionDefinition, Features, RequestClient } from '@segment/actions-core'
import type { ActionDefinition, RequestClient } from '@segment/actions-core'
import { PayloadValidationError, InvalidAuthenticationError, APIError } from '@segment/actions-core'
import type { Settings } from '../generated-types'
import type { Payload } from './generated-types'
@@ -96,7 +96,7 @@ const action: ActionDefinition<Settings, Payload> = {
}
},

perform: async (request, { payload, settings, features }) => {
perform: async (request, { payload, settings }) => {
// Check if client_identifier_id is valid
let rt_access_token_response
// Generate JWT token
@@ -111,9 +111,9 @@ const action: ActionDefinition<Settings, Payload> = {
}
const rt_access_token = rt_access_token_response?.data?.token
// Process the payload with the valid token
return process_payload(request, [payload], rt_access_token, settings.client_identifier_id, features || {})
return process_payload(request, [payload], rt_access_token, settings.client_identifier_id)
},
performBatch: async (request, { payload, settings, features }) => {
performBatch: async (request, { payload, settings }) => {
// Ensure client_identifier_id is provided
let rt_access_token_response
// Generate JWT token
@@ -128,7 +128,7 @@ const action: ActionDefinition<Settings, Payload> = {
}
const rt_access_token = rt_access_token_response?.data?.token
// Process the payload with the valid token
return process_payload(request, payload, rt_access_token, settings.client_identifier_id, features ?? {})
return process_payload(request, payload, rt_access_token, settings.client_identifier_id)
}
}

@@ -137,10 +137,9 @@ async function process_payload(
request: RequestClient,
payload: Payload[],
token: string | undefined,
client_identifier_id: string,
features: Features
client_identifier_id: string
) {
const body = gen_update_segment_payload(payload, client_identifier_id, features)
const body = gen_update_segment_payload(payload, client_identifier_id)
// Send request to Delivr AI only when all events in the batch include selected Ids
if (body.data.length > 0) {
return await request(`${DELIVRAI_BASE_URL}${DELIVRAI_SEGMENTATION_AUDIENCE}`, {
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Payload } from './updateSegment/generated-types'
import { DelivrAIPayload } from './types'
import { Features, RequestClient } from '@segment/actions-core'
import { RequestClient } from '@segment/actions-core'
import { DELIVRAI_BASE_URL, DELIVRAI_GET_TOKEN } from './constants'
import { processHashing } from '../../lib/hashing-utils'

@@ -50,11 +50,7 @@ export function validate_phone(phone: string) {
return ''
}
}
export function gen_update_segment_payload(
payloads: Payload[],
client_identifier_id: string,
features: Features
): DelivrAIPayload {
export function gen_update_segment_payload(payloads: Payload[], client_identifier_id: string): DelivrAIPayload {
const data_groups: {
[hashed_email: string]: {
exp: string
@@ -68,7 +64,7 @@ export function gen_update_segment_payload(
for (const event of payloads) {
let hashed_email: string | undefined = ''
if (event.email) {
hashed_email = processHashing(event.email.toLowerCase(), 'sha256', 'hex', features, 'actions-delivrai-audiences')
hashed_email = processHashing(event.email.toLowerCase(), 'sha256', 'hex')
}
let idfa: string | undefined = ''
let gpsaid: string | undefined = ''
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import { Features } from '@segment/actions-core'
import { processHashing } from '../../lib/hashing-utils'

export function hashAndEncode(property: string, features: Features): string {
return processHashing(property, 'sha256', 'hex', features, 'actions-dynamic-yield-audiences')
export function hashAndEncode(property: string): string {
return processHashing(property, 'sha256', 'hex')
}

export function hashAndEncodeToInt(property: string, features: Features): number {
const hash = processHashing(property, 'sha256', 'hex', features, 'actions-dynamic-yield-audiences')
export function hashAndEncodeToInt(property: string): number {
const hash = processHashing(property, 'sha256', 'hex')
const bigInt = BigInt('0x' + hash)
let integerString = bigInt.toString()
if (integerString.length > 16) {
Loading
Oops, something went wrong.
Loading
Oops, something went wrong.