diff --git a/backend/src/services/member/memberAttributesService.ts b/backend/src/services/member/memberAttributesService.ts index 07d3dcc91f..36597f5c85 100644 --- a/backend/src/services/member/memberAttributesService.ts +++ b/backend/src/services/member/memberAttributesService.ts @@ -2,7 +2,7 @@ import * as lodash from 'lodash' import { captureApiChange, memberEditProfileAction } from '@crowd/audit-logs' -import { Error404 } from '@crowd/common' +import { Error404, getAttributeValue, getCountry, hasAttributeValue } from '@crowd/common' import { deleteMemberBotSuggestion, deleteMemberNoBot, @@ -78,6 +78,18 @@ export default class MemberAttributesService extends LoggerBase { } } + if (!hasAttributeValue(data.country)) { + const location = getAttributeValue(data.location) + const country = getCountry(location) + if (country) { + data.country = { + ...data.country, + system: country, + default: country, + } + } + } + await updateMemberAttributes(qx, memberId, data) // Handle isBot status and maintain consistency with bot tracking tables diff --git a/backend/src/services/memberService.ts b/backend/src/services/memberService.ts index a9b1fdd164..4d02949141 100644 --- a/backend/src/services/memberService.ts +++ b/backend/src/services/memberService.ts @@ -4,7 +4,15 @@ import moment from 'moment-timezone' import validator from 'validator' import { captureApiChange, memberUnmergeAction } from '@crowd/audit-logs' -import { Error400, calculateReach, getProperDisplayName, isDomainExcluded } from '@crowd/common' +import { + Error400, + calculateReach, + getAttributeValue, + getCountry, + getProperDisplayName, + hasAttributeValue, + isDomainExcluded, +} from '@crowd/common' import { CommonMemberService, getGithubInstallationToken, @@ -446,6 +454,17 @@ export default class MemberService extends LoggerBase { const toUpdate = CommonMemberService.membersMerge(existing, data) if (toUpdate.attributes) { + if (!hasAttributeValue(toUpdate.attributes.country)) { + const location = getAttributeValue(toUpdate.attributes.location) + const country = getCountry(location) + if (country) { + toUpdate.attributes.country = { + ...toUpdate.attributes.country, + system: country, + } + } + } + toUpdate.attributes = await this.setAttributesDefaultValues(toUpdate.attributes) } @@ -459,6 +478,17 @@ export default class MemberService extends LoggerBase { // It is important to call it with doPopulateRelations=false // because otherwise the performance is greatly decreased in integrations if (data.attributes) { + if (!hasAttributeValue(data.attributes.country)) { + const location = getAttributeValue(data.attributes.location) + const country = getCountry(location) + if (country) { + data.attributes.country = { + ...data.attributes.country, + system: country, + } + } + } + data.attributes = await this.setAttributesDefaultValues(data.attributes) } @@ -789,6 +819,20 @@ export default class MemberService extends LoggerBase { data.displayName = getProperDisplayName(data.displayName) } + if (data.attributes) { + if (!hasAttributeValue(data.attributes.country)) { + const location = getAttributeValue(data.attributes.location) + const country = getCountry(location) + if (country) { + data.attributes.country = { + ...data.attributes.country, + system: country, + default: data.attributes.country?.default ?? country, + } + } + } + } + const record = await MemberRepository.update(id, data, repoOptions, { manualChange, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7247b9fafe..913979af01 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2085,6 +2085,9 @@ importers: config: specifier: ^3.3.9 version: 3.3.11 + i18n-iso-countries: + specifier: ^7.14.0 + version: 7.14.0 lodash.clonedeep: specifier: ^4.5.0 version: 4.5.0 @@ -6393,6 +6396,9 @@ packages: dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} + diacritics@1.3.0: + resolution: {integrity: sha512-wlwEkqcsaxvPJML+rDh/2iS824jbREk6DUMUKkEaSlxdYHeS43cClJtsWglvw2RfeXGm6ohKDqsXteJ5sP5enA==} + diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -7497,6 +7503,10 @@ packages: resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} engines: {node: '>=10.18'} + i18n-iso-countries@7.14.0: + resolution: {integrity: sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg==} + engines: {node: '>= 12'} + iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -16122,6 +16132,8 @@ snapshots: asap: 2.0.6 wrappy: 1.0.2 + diacritics@1.3.0: {} + diff@4.0.2: {} dir-glob@3.0.1: @@ -17646,6 +17658,10 @@ snapshots: hyperdyperid@1.2.0: {} + i18n-iso-countries@7.14.0: + dependencies: + diacritics: 1.3.0 + iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2 diff --git a/services/apps/data_sink_worker/src/service/member.service.ts b/services/apps/data_sink_worker/src/service/member.service.ts index fc3b7bb4ff..d66c62d8ee 100644 --- a/services/apps/data_sink_worker/src/service/member.service.ts +++ b/services/apps/data_sink_worker/src/service/member.service.ts @@ -5,8 +5,11 @@ import uniqby from 'lodash.uniqby' import { ApplicationError, DEFAULT_TENANT_ID, + getAttributeValue, + getCountry, getEarliestValidDate, getProperDisplayName, + hasAttributeValue, isDomainExcluded, isObjectEmpty, isSameMemberIdentity, @@ -331,6 +334,14 @@ export default class MemberService extends LoggerBase { 'memberService -> create -> validateAttributes', ) + if (!hasAttributeValue(attributes.country)) { + const location = getAttributeValue(attributes.location) + const country = getCountry(location) + if (country) { + attributes.country = Object.assign({}, attributes.country, { system: country }) + } + } + attributes = await logExecutionTimeV2( () => memberAttributeService.setAttributesDefaultValues(attributes), this.log, @@ -592,6 +603,17 @@ export default class MemberService extends LoggerBase { if (toUpdate.attributes) { this.log.trace({ memberId: id }, 'Setting attribute default values!') + + if (!hasAttributeValue(toUpdate.attributes.country)) { + const location = getAttributeValue(toUpdate.attributes.location) + const country = getCountry(location) + if (country) { + toUpdate.attributes.country = Object.assign({}, toUpdate.attributes.country, { + system: country, + }) + } + } + toUpdate.attributes = await logExecutionTimeV2( () => memberAttributeService.setAttributesDefaultValues(toUpdate.attributes), this.log, diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts index 1ffb8ffbc3..1d494e3e8a 100644 --- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts @@ -4,6 +4,9 @@ import _ from 'lodash' import { generateUUIDv1, + getAttributeValue, + getCountry, + hasAttributeValue, hasIntersection, replaceDoubleQuotes, sanitizeMemberOrganizationDateRange, @@ -45,6 +48,7 @@ import { refreshMaterializedView } from '@crowd/data-access-layer/src/utils' import { SearchSyncApiClient } from '@crowd/opensearch' import { RedisCache } from '@crowd/redis' import { + IAttributes, IEnrichableMember, IEnrichableMemberIdentityActivityAggregate, IMemberEnrichmentCache, @@ -347,7 +351,7 @@ export async function updateMemberUsingSquashedPayload( } // process attributes - let attributes = existingMemberData.attributes as Record + let attributes = existingMemberData.attributes as IAttributes if (squashedPayload.attributes) { svc.log.debug({ memberId }, 'Updating member attributes!') @@ -355,8 +359,20 @@ export async function updateMemberUsingSquashedPayload( attributes = _.merge({}, attributes, squashedPayload.attributes) if (Object.keys(attributes).length > 0) { + // Infer country from location when no country source is set. + if (!hasAttributeValue(attributes.country)) { + const location = getAttributeValue(attributes.location) + const country = getCountry(location) + if (country) { + attributes.country = { + ...attributes.country, + system: country, + } + } + } + const priorities = await getPriorityArray() - attributes = await setAttributesDefaultValues(attributes, priorities) + attributes = (await setAttributesDefaultValues(attributes, priorities)) as IAttributes } didUpdate = true await updateMemberAttributes(qx, memberId, attributes) diff --git a/services/libs/common/package.json b/services/libs/common/package.json index a17df50292..1626417273 100644 --- a/services/libs/common/package.json +++ b/services/libs/common/package.json @@ -16,6 +16,7 @@ "dependencies": { "@crowd/types": "workspace:*", "config": "^3.3.9", + "i18n-iso-countries": "^7.14.0", "lodash.clonedeep": "^4.5.0", "lodash.get": "~4.4.2", "lodash.isarray": "^4.0.0", diff --git a/services/libs/common/src/constants/index.ts b/services/libs/common/src/constants/index.ts index 960988b523..bfe59f50c2 100644 --- a/services/libs/common/src/constants/index.ts +++ b/services/libs/common/src/constants/index.ts @@ -1,3 +1,4 @@ export * from './email-providers' export * from './bots' export * from './disposable-email-domains' +export * from './location' diff --git a/services/libs/common/src/constants/location.ts b/services/libs/common/src/constants/location.ts new file mode 100644 index 0000000000..454aae750b --- /dev/null +++ b/services/libs/common/src/constants/location.ts @@ -0,0 +1,108 @@ +/** + * US state / district names (not postal codes). + * Excludes georgia and washington — too easy to confuse with other places. + */ +export const US_STATE_NAMES = new Set([ + 'alabama', + 'alaska', + 'arizona', + 'arkansas', + 'california', + 'colorado', + 'connecticut', + 'delaware', + 'florida', + 'hawaii', + 'idaho', + 'illinois', + 'indiana', + 'iowa', + 'kansas', + 'kentucky', + 'louisiana', + 'maine', + 'maryland', + 'massachusetts', + 'michigan', + 'minnesota', + 'mississippi', + 'missouri', + 'montana', + 'nebraska', + 'nevada', + 'new hampshire', + 'new jersey', + 'new mexico', + 'new york', + 'north carolina', + 'north dakota', + 'ohio', + 'oklahoma', + 'oregon', + 'pennsylvania', + 'rhode island', + 'south carolina', + 'south dakota', + 'tennessee', + 'texas', + 'utah', + 'vermont', + 'virginia', + 'west virginia', + 'wisconsin', + 'wyoming', + 'district of columbia', +]) + +/** Common alternate spellings → a name the country package recognizes. */ +export const COUNTRY_NAME_INPUT_ALIASES = new Map([ + ['brasil', 'Brazil'], + ['turkiye', 'Türkiye'], + ['viet nam', 'Vietnam'], + ['korea', 'South Korea'], + ['england', 'United Kingdom'], + ['scotland', 'United Kingdom'], + ['wales', 'United Kingdom'], +]) + +/** Tokens that should never resolve to a country. */ +export const JUNK_LOCATION_TOKENS = new Set([ + 'earth', + 'mars', + 'moon', + 'remote', + 'worldwide', + 'world', + 'global', + 'internet', + 'somewhere', + 'nowhere', + 'unknown', + 'n/a', + 'na', + 'null', + 'undefined', + 'home', + 'here', + 'africa', + 'europe', + 'asia', + 'antarctica', +]) + +/** Names that could mean more than one place — skip instead of guessing. */ +export const AMBIGUOUS_LOCATION_TOKENS = new Set(['georgia', 'washington']) + +/** Preferred display names for a few ISO codes with awkward official forms. */ +export const COUNTRY_DISPLAY_NAME_BY_ALPHA2 = new Map([ + ['US', 'United States'], + ['GB', 'United Kingdom'], + ['CN', 'China'], + ['TW', 'Taiwan'], + ['RU', 'Russia'], + ['TR', 'Turkey'], + ['KR', 'South Korea'], + ['NL', 'Netherlands'], + ['CZ', 'Czech Republic'], + ['AE', 'United Arab Emirates'], +]) diff --git a/services/libs/common/src/country.ts b/services/libs/common/src/country.ts new file mode 100644 index 0000000000..d91c8a6bf8 --- /dev/null +++ b/services/libs/common/src/country.ts @@ -0,0 +1,88 @@ +import countries from 'i18n-iso-countries' + +import { + AMBIGUOUS_LOCATION_TOKENS, + COUNTRY_DISPLAY_NAME_BY_ALPHA2, + COUNTRY_NAME_INPUT_ALIASES, + JUNK_LOCATION_TOKENS, + US_STATE_NAMES, +} from './constants' + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, ' ').trim() +} + +function normalizeToken(value: string): string { + return normalizeWhitespace(value) + .replace(/[.!?]+$/, '') + .toLowerCase() +} + +function toDisplayName(alpha2: string): string | undefined { + const override = COUNTRY_DISPLAY_NAME_BY_ALPHA2.get(alpha2) + if (override) { + return override + } + + const official = countries.getName(alpha2, 'en', { select: 'official' }) + const alias = countries.getName(alpha2, 'en', { select: 'alias' }) + + // Prefer plain aliases (e.g. China) over awkward official forms; skip short ones like "UK". + if (alias && alias.length >= 4 && !alias.includes(',') && !alias.includes('.')) { + return alias + } + + return official || undefined +} + +function parseCountryToken(token: string): string | undefined { + if (!token || JUNK_LOCATION_TOKENS.has(token) || AMBIGUOUS_LOCATION_TOKENS.has(token)) { + return undefined + } + + if (US_STATE_NAMES.has(token)) { + return 'United States' + } + + const aliased = COUNTRY_NAME_INPUT_ALIASES.get(token) || token + + // Name/alias lookup only — getAlpha2Code does not treat CA/IN/DE as countries. + const alpha2 = countries.getAlpha2Code(aliased, 'en') + if (!alpha2) { + return undefined + } + + return toDisplayName(alpha2) +} + +/** + * Resolves a country name from a location string. + */ +export function getCountry(location: string | null | undefined): string | undefined { + if (!location) { + return undefined + } + + const normalized = normalizeWhitespace(location) + if (!normalized || JUNK_LOCATION_TOKENS.has(normalized.toLowerCase())) { + return undefined + } + + // Prefer later segments (…, Country) but also accept Country, City. + const parts = normalized + .replaceAll(' - ', ',') + .replaceAll('/', ',') + .replaceAll('|', ',') + .split(',') + .map((part) => normalizeToken(part)) + .filter(Boolean) + + for (let i = parts.length - 1; i >= 0; i--) { + const country = parseCountryToken(parts[i]) + if (country) { + return country + } + } + + return undefined +} diff --git a/services/libs/common/src/index.ts b/services/libs/common/src/index.ts index 078bbaadfb..6a4eb89884 100644 --- a/services/libs/common/src/index.ts +++ b/services/libs/common/src/index.ts @@ -35,6 +35,7 @@ export * from './rawQueryParser' export * from './byteLength' export * from './domain' export * from './displayName' +export * from './country' export * from './jira' export * from './email' export * from './bot' diff --git a/services/libs/common/src/member.ts b/services/libs/common/src/member.ts index 5a9c614732..f6c7524ba2 100644 --- a/services/libs/common/src/member.ts +++ b/services/libs/common/src/member.ts @@ -64,6 +64,38 @@ export function normalizeMemberIdentities< return Array.from(seen.values()) } +/** Prefer `default`, else first non-empty string source value. */ +export function getAttributeValue(attribute: unknown): string | undefined { + if (!attribute || typeof attribute !== 'object') { + return undefined + } + + const record = attribute as Record + + if (typeof record.default === 'string' && record.default.trim()) { + return record.default + } + + for (const [key, value] of Object.entries(record)) { + if (key === 'default') continue + if (typeof value === 'string' && value.trim()) { + return value + } + } + + return undefined +} + +export function hasAttributeValue(attribute: unknown): boolean { + if (!attribute || typeof attribute !== 'object') { + return false + } + + return Object.values(attribute as Record).some( + (v) => typeof v === 'string' && v.trim().length > 0, + ) +} + export async function setAttributesDefaultValues( attributes: Record, priorities: string[],