From ad329fa736f45d1afb1458945b11f719ca683fc2 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 6 Nov 2024 16:08:27 +0100 Subject: [PATCH 01/13] crustdata linkedin scraper start --- .../members_enrichment_worker/src/factory.ts | 3 + .../src/sources/crustdata/service.ts | 326 ++++++++++++++++++ .../src/sources/crustdata/types.ts | 42 +++ .../progai-linkedin-scraper/service.ts | 106 +++--- .../sources/progai-linkedin-scraper/types.ts | 6 +- .../members_enrichment_worker/src/types.ts | 15 +- .../src/workflows/getMembersToEnrich.ts | 1 + services/libs/common/src/strings.ts | 4 + services/libs/types/src/enums/enrichment.ts | 1 + services/libs/types/src/enums/members.ts | 1 + .../libs/types/src/enums/organizations.ts | 1 + 11 files changed, 448 insertions(+), 58 deletions(-) create mode 100644 services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts create mode 100644 services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts diff --git a/services/apps/premium/members_enrichment_worker/src/factory.ts b/services/apps/premium/members_enrichment_worker/src/factory.ts index 6ea8143162..754df359cf 100644 --- a/services/apps/premium/members_enrichment_worker/src/factory.ts +++ b/services/apps/premium/members_enrichment_worker/src/factory.ts @@ -3,6 +3,7 @@ import { Logger } from '@crowd/logging' import { MemberEnrichmentSource } from '@crowd/types' import EnrichmentServiceClearbit from './sources/clearbit/service' +import EnrichmentServiceCrustdata from './sources/crustdata/service' import EnrichmentServiceProgAILinkedinScraper from './sources/progai-linkedin-scraper/service' import EnrichmentServiceProgAI from './sources/progai/service' import EnrichmentServiceSerpApi from './sources/serp/service' @@ -27,6 +28,8 @@ export class EnrichmentSourceServiceFactory { return new EnrichmentServiceSerpApi(log) case MemberEnrichmentSource.PROGAI_LINKEDIN_SCRAPER: return new EnrichmentServiceProgAILinkedinScraper(log) + case MemberEnrichmentSource.CRUSTDATA: + return new EnrichmentServiceCrustdata(log) default: throw new Error(`Enrichment service for ${source} is not found!`) } diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts new file mode 100644 index 0000000000..fe7c851a74 --- /dev/null +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -0,0 +1,326 @@ +import axios from 'axios' + +import { isValidEmail } from '@crowd/common' +import { Logger, LoggerBase } from '@crowd/logging' +import { + IMemberEnrichmentCache, + IMemberIdentity, + MemberAttributeName, + MemberEnrichmentSource, + MemberIdentityType, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { findMemberEnrichmentCacheForAllSources } from '../../activities/enrichment' +import { EnrichmentSourceServiceFactory } from '../../factory' +import { + IEnrichmentService, + IEnrichmentSourceInput, + IMemberEnrichmentAttributeSettings, + IMemberEnrichmentData, + IMemberEnrichmentDataNormalized, +} from '../../types' +import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common' + +import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types' + +export default class EnrichmentServiceCrustdata extends LoggerBase implements IEnrichmentService { + public source: MemberEnrichmentSource = MemberEnrichmentSource.CRUSTDATA + public platform = `enrichment-${this.source}` + + public alsoFindInputsInSourceCaches: MemberEnrichmentSource[] = [ + MemberEnrichmentSource.PROGAI, + MemberEnrichmentSource.CLEARBIT, + MemberEnrichmentSource.SERP, + ] + + public enrichMembersWithActivityMoreThan = 200 + + public enrichableBySql = `("activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan}) AND mi.verified AND mi.type = 'username' and mi.platform = 'linkedin'` + + public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 90 + + public attributeSettings: IMemberEnrichmentAttributeSettings = { + [MemberAttributeName.AVATAR_URL]: { + fields: ['profile_picture_url'], + }, + [MemberAttributeName.JOB_TITLE]: { + fields: ['title'], + }, + [MemberAttributeName.BIO]: { + fields: ['summary', 'headline'], + }, + [MemberAttributeName.SKILLS]: { + fields: ['skills'], + transform: (skills: string) => skills.split(',').sort(), + }, + [MemberAttributeName.LANGUAGES]: { + fields: ['languages'], + transform: (languages: string[]) => languages.sort(), + }, + [MemberAttributeName.SCHOOLS]: { + fields: ['all_schools'], + transform: (schools: string[]) => schools.sort(), + }, + } + + constructor(public readonly log: Logger) { + super(log) + } + + async isEnrichableBySource(input: IEnrichmentSourceInput): Promise { + const caches = await findMemberEnrichmentCacheForAllSources(input.memberId) + + let hasEnrichableLinkedinInCache = false + for (const cache of caches) { + if (this.alsoFindInputsInSourceCaches.includes(cache.source)) { + const service = EnrichmentSourceServiceFactory.getEnrichmentSourceService( + cache.source, + this.log, + ) + const normalized = service.normalize(cache.data) as IMemberEnrichmentDataNormalized + if (normalized.identities.some((i) => i.platform === PlatformType.LINKEDIN)) { + hasEnrichableLinkedinInCache = true + break + } + + break + } + } + + return ( + input.activityCount > this.enrichMembersWithActivityMoreThan && + (hasEnrichableLinkedinInCache || + (input.linkedin && input.linkedin.value && input.linkedin.verified)) + ) + } + + async hasRemainingCredits(): Promise { + try { + const config = { + method: 'get', + url: `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/user/credits`, + headers: { + Authorization: `Token ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + }, + } + + const response: IMemberEnrichmentCrustdataRemainingCredits = (await axios(config)).data + + return response.credits > 5 + } catch (error) { + this.log.error('Error while checking serpapi account usage', error) + return false + } + } + + async getData(input: IEnrichmentSourceInput): Promise { + const profiles: IMemberEnrichmentDataCrustdata[] = [] + const caches = await findMemberEnrichmentCacheForAllSources(input.memberId) + + const consumableIdentities = await this.findDistinctScrapableLinkedinIdentities(input, caches) + + for (const identity of consumableIdentities) { + const data = await this.getDataUsingLinkedinHandle(identity.value) + if (data) { + profiles.push({ + ...data, + metadata: { + repeatedTimesInDifferentSources: identity.repeatedTimesInDifferentSources, + isFromVerifiedSource: identity.isFromVerifiedSource, + }, + }) + } + } + + return profiles.length > 0 ? profiles : null + } + + private async getDataUsingLinkedinHandle( + handle: string, + ): Promise { + const url = `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/screener/person/enrich` + const config = { + method: 'get', + url, + params: { + linkedin_profile_url: `https://linkedin.com/in/${handle}`, + enrich_realtime: false, + }, + headers: { + Authorization: `Token ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + }, + } + + const response: IMemberEnrichmentDataCrustdata[] = (await axios(config)).data + + return response.length > 0 ? response[0] : null + } + + private async findDistinctScrapableLinkedinIdentities( + input: IEnrichmentSourceInput, + caches: IMemberEnrichmentCache[], + ): Promise< + (IMemberIdentity & { repeatedTimesInDifferentSources: number; isFromVerifiedSource: boolean })[] + > { + const consumableIdentities: (IMemberIdentity & { + repeatedTimesInDifferentSources: number + isFromVerifiedSource: boolean + })[] = [] + const linkedinUrlHashmap = new Map() + + for (const cache of caches) { + if (this.alsoFindInputsInSourceCaches.includes(cache.source)) { + const service = EnrichmentSourceServiceFactory.getEnrichmentSourceService( + cache.source, + this.log, + ) + const normalized = service.normalize(cache.data) as IMemberEnrichmentDataNormalized + if (normalized.identities.some((i) => i.platform === PlatformType.LINKEDIN)) { + const identity = normalized.identities.find((i) => i.platform === PlatformType.LINKEDIN) + if (!linkedinUrlHashmap.get(identity.value)) { + consumableIdentities.push({ + ...identity, + repeatedTimesInDifferentSources: 1, + isFromVerifiedSource: false, + }) + linkedinUrlHashmap.set(identity.value, 1) + } else { + const repeatedTimesInDifferentSources = linkedinUrlHashmap.get(identity.value) + 1 + linkedinUrlHashmap.set(identity.value, repeatedTimesInDifferentSources) + consumableIdentities.find( + (i) => i.value === identity.value, + ).repeatedTimesInDifferentSources = repeatedTimesInDifferentSources + } + } + } + } + + if (input.linkedin && input.linkedin.value && input.linkedin.verified) { + if (!linkedinUrlHashmap.get(input.linkedin.value)) { + consumableIdentities.push({ + ...input.linkedin, + repeatedTimesInDifferentSources: 1, + isFromVerifiedSource: true, + }) + } else { + const repeatedTimesInDifferentSources = linkedinUrlHashmap.get(input.linkedin.value) + 1 + const identityFound = consumableIdentities.find((i) => i.value === input.linkedin.value) + + identityFound.repeatedTimesInDifferentSources = repeatedTimesInDifferentSources + identityFound.isFromVerifiedSource = true + } + } + return consumableIdentities + } + + normalize(profiles: IMemberEnrichmentDataCrustdata[]): IMemberEnrichmentDataNormalized[] { + const normalizedProfiles: IMemberEnrichmentDataNormalized[] = [] + + for (const profile of profiles) { + const profileNormalized = this.normalizeOneResult(profile) + normalizedProfiles.push({ ...profileNormalized, metadata: profile.metadata }) + } + + return normalizedProfiles.length > 0 ? normalizedProfiles : null + } + + private normalizeOneResult( + data: IMemberEnrichmentDataCrustdata, + ): IMemberEnrichmentDataNormalized { + let normalized: IMemberEnrichmentDataNormalized = { + identities: [], + attributes: {}, + contributions: [], + memberOrganizations: [], + reach: {}, + } + + normalized = this.normalizeIdentities(data, normalized) + normalized = normalizeAttributes(data, normalized, this.attributeSettings, this.platform) + normalized = this.normalizeEmployment(data, normalized) + + if (data.num_of_connections) { + normalized.reach[this.platform] = data.num_of_connections + } + + return normalized + } + + private normalizeIdentities( + data: IMemberEnrichmentDataCrustdata, + normalized: IMemberEnrichmentDataNormalized, + ): IMemberEnrichmentDataNormalized { + if (!normalized.identities) { + normalized.identities = [] + } + + if (!normalized.attributes) { + normalized.attributes = {} + } + + if (data.name) { + normalized.displayName = data.name + } + + if (data.email) { + for (const email of data.email.split(',').filter(isValidEmail)) { + normalized.identities.push({ + type: MemberIdentityType.EMAIL, + platform: this.platform, + value: email.trim(), + verified: false, + }) + } + } + + if (data.twitter_handle) { + normalized = normalizeSocialIdentity( + { + handle: data.twitter_handle, + platform: PlatformType.TWITTER, + }, + MemberIdentityType.USERNAME, + true, + normalized, + ) + } + + return normalized + } + + private normalizeEmployment( + data: IMemberEnrichmentDataCrustdata, + normalized: IMemberEnrichmentDataNormalized, + ): IMemberEnrichmentDataNormalized { + const employmentInformation = data.past_employers.concat(data.current_employers) + if (data.past_employers) { + for (const workExperience of employmentInformation) { + const identities = [] + + if (workExperience.employer_linkedin_id) { + identities.push({ + platform: PlatformType.LINKEDIN, + value: `company:${workExperience.employer_linkedin_id}`, + type: OrganizationIdentityType.USERNAME, + verified: true, + }) + } + + normalized.memberOrganizations.push({ + name: workExperience.employer_name, + source: OrganizationSource.ENRICHMENT_CRUSTDATA, + identities, + title: workExperience.employee_title, + startDate: workExperience.start_date, + endDate: workExperience.end_date, + organizationDescription: workExperience.employer_linkedin_description, + }) + } + } + + return normalized + } +} diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts new file mode 100644 index 0000000000..a88c97aa5d --- /dev/null +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts @@ -0,0 +1,42 @@ +import { IMemberEnrichmentLinkedinScraperMetadata } from '../../types' + +export interface IMemberEnrichmentCrustdataEmployer { + employer_name: string + employer_linkedin_id: string + employer_linkedin_description?: string + employer_company_id: number[] + employee_position_id?: number + employee_title: string + employee_description?: string + employee_location: string + start_date: string + end_date: string +} + +export interface IMemberEnrichmentDataCrustdata { + linkedin_profile_url: string + linkedin_flagship_url: string + name: string + email: string + title: string + last_updated: string + headline: string + summary: string + num_of_connections: number + skills: string + profile_picture_url: string + twitter_handle: string + languages: string[] + all_employers: string[] + past_employers: IMemberEnrichmentCrustdataEmployer[] + current_employers: IMemberEnrichmentCrustdataEmployer[] + all_employers_company_id: number[] + all_titles: string[] + all_schools: string[] + all_degrees: string[] + metadata: IMemberEnrichmentLinkedinScraperMetadata +} + +export interface IMemberEnrichmentCrustdataRemainingCredits { + credits: number +} diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts index 29f95d4187..8793565ebb 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts @@ -72,7 +72,59 @@ export default class EnrichmentServiceProgAILinkedinScraper return true } - private async findConsumableLinkedinIdentities( + async getData( + input: IEnrichmentSourceInput, + ): Promise { + const profiles: IMemberEnrichmentDataProgAILinkedinScraper[] = [] + const caches = await findMemberEnrichmentCacheForAllSources(input.memberId) + + const consumableIdentities = await this.findDistinctScrapableLinkedinIdentities(input, caches) + + for (const identity of consumableIdentities) { + const data = await this.getDataUsingLinkedinHandle(identity.value) + if (data) { + const existingProgaiCache = caches.find((c) => c.source === MemberEnrichmentSource.PROGAI) + // we don't want to reinforce the cache with the same data, only save to cache + // if a new profile is returned from progai + if ( + existingProgaiCache && + existingProgaiCache.data && + (existingProgaiCache.data as IMemberEnrichmentDataProgAI).id == data.id + ) { + continue + } + profiles.push({ + ...data, + metadata: { + repeatedTimesInDifferentSources: identity.repeatedTimesInDifferentSources, + isFromVerifiedSource: identity.isFromVerifiedSource, + }, + }) + } + } + + return profiles.length > 0 ? profiles : null + } + + private async getDataUsingLinkedinHandle(handle: string): Promise { + const url = `${process.env['CROWD_ENRICHMENT_PROGAI_URL']}/get_profile` + const config = { + method: 'get', + url, + params: { + linkedin_url: `https://linkedin.com/in/${handle}`, + with_emails: true, + api_key: process.env['CROWD_ENRICHMENT_PROGAI_API_KEY'], + }, + headers: {}, + } + + const response: IMemberEnrichmentDataProgAIResponse = (await axios(config)).data + + return response?.profile || null + } + + private async findDistinctScrapableLinkedinIdentities( input: IEnrichmentSourceInput, caches: IMemberEnrichmentCache[], ): Promise< @@ -130,58 +182,6 @@ export default class EnrichmentServiceProgAILinkedinScraper return consumableIdentities } - async getData( - input: IEnrichmentSourceInput, - ): Promise { - const profiles: IMemberEnrichmentDataProgAILinkedinScraper[] = [] - const caches = await findMemberEnrichmentCacheForAllSources(input.memberId) - - const consumableIdentities = await this.findConsumableLinkedinIdentities(input, caches) - - for (const identity of consumableIdentities) { - const data = await this.getDataUsingLinkedinHandle(identity.value) - if (data) { - const existingProgaiCache = caches.find((c) => c.source === MemberEnrichmentSource.PROGAI) - // we don't want to reinforce the cache with the same data, only save to cache - // if a new profile is returned from progai - if ( - existingProgaiCache && - existingProgaiCache.data && - (existingProgaiCache.data as IMemberEnrichmentDataProgAI).id == data.id - ) { - continue - } - profiles.push({ - ...data, - metadata: { - repeatedTimesInDifferentSources: identity.repeatedTimesInDifferentSources, - isFromVerifiedSource: identity.isFromVerifiedSource, - }, - }) - } - } - - return profiles.length > 0 ? profiles : null - } - - private async getDataUsingLinkedinHandle(handle: string): Promise { - const url = `${process.env['CROWD_ENRICHMENT_PROGAI_URL']}/get_profile` - const config = { - method: 'get', - url, - params: { - linkedin_url: `https://linkedin.com/in/${handle}`, - with_emails: true, - api_key: process.env['CROWD_ENRICHMENT_PROGAI_API_KEY'], - }, - headers: {}, - } - - const response: IMemberEnrichmentDataProgAIResponse = (await axios(config)).data - - return response?.profile || null - } - normalize( profiles: IMemberEnrichmentDataProgAILinkedinScraper[], ): IMemberEnrichmentDataNormalized[] { diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/types.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/types.ts index af7bda13c7..3b36f81906 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/types.ts @@ -1,8 +1,6 @@ +import { IMemberEnrichmentLinkedinScraperMetadata } from '../../types' import { IMemberEnrichmentDataProgAI } from '../progai/types' export interface IMemberEnrichmentDataProgAILinkedinScraper extends IMemberEnrichmentDataProgAI { - metadata: { - repeatedTimesInDifferentSources: number - isFromVerifiedSource: boolean - } + metadata: IMemberEnrichmentLinkedinScraperMetadata } diff --git a/services/apps/premium/members_enrichment_worker/src/types.ts b/services/apps/premium/members_enrichment_worker/src/types.ts index 4229dea19e..fe706eacd2 100644 --- a/services/apps/premium/members_enrichment_worker/src/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/types.ts @@ -2,6 +2,7 @@ import { IAttributes, IMemberContribution, IMemberIdentity, + IMemberReach, IOrganizationIdentity, MemberAttributeName, MemberEnrichmentSource, @@ -10,6 +11,7 @@ import { } from '@crowd/types' import { IMemberEnrichmentDataClearbit } from './sources/clearbit/types' +import { IMemberEnrichmentDataCrustdata } from './sources/crustdata/types' import { IMemberEnrichmentDataProgAILinkedinScraper } from './sources/progai-linkedin-scraper/types' import { IMemberEnrichmentDataProgAI } from './sources/progai/types' import { IMemberEnrichmentDataSerp } from './sources/serp/types' @@ -30,6 +32,8 @@ export type IMemberEnrichmentData = | IMemberEnrichmentDataClearbit | IMemberEnrichmentDataSerp | IMemberEnrichmentDataProgAILinkedinScraper[] + | IMemberEnrichmentDataCrustdata + | IMemberEnrichmentDataCrustdata[] export interface IEnrichmentService { source: MemberEnrichmentSource @@ -57,24 +61,33 @@ export interface IEnrichmentService { ): IMemberEnrichmentDataNormalized | IMemberEnrichmentDataNormalized[] } +export type IMemberEnrichmentMetadataNormalized = IMemberEnrichmentLinkedinScraperMetadata + export interface IMemberEnrichmentDataNormalized { identities?: IMemberIdentity[] contributions?: IMemberContribution[] attributes?: IAttributes + reach?: IMemberReach memberOrganizations?: IMemberEnrichmentDataNormalizedOrganization[] displayName?: string - metadata?: Record + metadata?: IMemberEnrichmentMetadataNormalized } export interface IMemberEnrichmentDataNormalizedOrganization { name: string identities?: IOrganizationIdentity[] title?: string + organizationDescription?: string startDate?: string endDate?: string source: OrganizationSource } +export interface IMemberEnrichmentLinkedinScraperMetadata { + repeatedTimesInDifferentSources: number + isFromVerifiedSource: boolean +} + export interface IGetMembersForEnrichmentArgs { afterCursor: { activityCount: number; memberId: string } | null } diff --git a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts index 5b4a639e65..6124f83223 100644 --- a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts +++ b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts @@ -25,6 +25,7 @@ export async function getMembersToEnrich(args: IGetMembersForEnrichmentArgs): Pr MemberEnrichmentSource.CLEARBIT, MemberEnrichmentSource.SERP, MemberEnrichmentSource.PROGAI_LINKEDIN_SCRAPER, + MemberEnrichmentSource.CRUSTDATA, ] const members = await getEnrichableMembers(MEMBER_ENRICHMENT_PER_RUN, sources, afterCursor) diff --git a/services/libs/common/src/strings.ts b/services/libs/common/src/strings.ts index 955497d01c..3c72f69be5 100644 --- a/services/libs/common/src/strings.ts +++ b/services/libs/common/src/strings.ts @@ -6,3 +6,7 @@ export const getCleanString = (value: string): string => { .toLowerCase() .trim() } + +export const isValidEmail = (email: string): boolean => { + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email.trim()) +} diff --git a/services/libs/types/src/enums/enrichment.ts b/services/libs/types/src/enums/enrichment.ts index b3c939d67d..7e46da08e7 100644 --- a/services/libs/types/src/enums/enrichment.ts +++ b/services/libs/types/src/enums/enrichment.ts @@ -3,4 +3,5 @@ export enum MemberEnrichmentSource { CLEARBIT = 'clearbit', SERP = 'serp', PROGAI_LINKEDIN_SCRAPER = 'progai-linkedin-scraper', + CRUSTDATA = 'crustdata', } diff --git a/services/libs/types/src/enums/members.ts b/services/libs/types/src/enums/members.ts index 84e2708bd2..31ac7805ba 100644 --- a/services/libs/types/src/enums/members.ts +++ b/services/libs/types/src/enums/members.ts @@ -36,6 +36,7 @@ export enum MemberAttributeName { COUNTRY = 'country', YEARS_OF_EXPERIENCE = 'yearsOfExperience', EDUCATION = 'education', + SCHOOLS = 'schools', AWARDS = 'awards', CERTIFICATIONS = 'certifications', WORK_EXPERIENCES = 'workExperiences', diff --git a/services/libs/types/src/enums/organizations.ts b/services/libs/types/src/enums/organizations.ts index 6261cd4b64..34dfd708cd 100644 --- a/services/libs/types/src/enums/organizations.ts +++ b/services/libs/types/src/enums/organizations.ts @@ -9,6 +9,7 @@ export enum OrganizationSource { EMAIL_DOMAIN = 'email-domain', ENRICHMENT_PROGAI = 'enrichment-progai', ENRICHMENT_CLEARBIT = 'enrichment-clearbit', + ENRICHMENT_CRUSTDATA = 'enrichment-crustdata', HUBSPOT = 'hubspot', GITHUB = 'github', UI = 'ui', From 3fb0180d60b68e50e2c2a759bf30a24d32227fd2 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 6 Nov 2024 16:20:05 +0100 Subject: [PATCH 02/13] update activity count requirement --- .../members_enrichment_worker/src/sources/crustdata/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index fe7c851a74..b4db2979f0 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -36,7 +36,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE MemberEnrichmentSource.SERP, ] - public enrichMembersWithActivityMoreThan = 200 + public enrichMembersWithActivityMoreThan = 1000 public enrichableBySql = `("activitySummary".total_count > ${this.enrichMembersWithActivityMoreThan}) AND mi.verified AND mi.type = 'username' and mi.platform = 'linkedin'` From c6c7b9e4c5a62fa4b20758212e0e0e608199a540 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 09:59:48 +0100 Subject: [PATCH 03/13] review points --- .../src/sources/crustdata/service.ts | 10 ++++------ .../src/sources/progai-linkedin-scraper/service.ts | 12 ++++-------- services/libs/common/src/strings.ts | 4 ---- services/libs/types/src/enums/members.ts | 4 ++++ 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index b4db2979f0..11c50d91bd 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -1,6 +1,6 @@ import axios from 'axios' -import { isValidEmail } from '@crowd/common' +import { isEmail } from '@crowd/common' import { Logger, LoggerBase } from '@crowd/logging' import { IMemberEnrichmentCache, @@ -85,8 +85,6 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE hasEnrichableLinkedinInCache = true break } - - break } } @@ -111,7 +109,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE return response.credits > 5 } catch (error) { - this.log.error('Error while checking serpapi account usage', error) + this.log.error('Error while checking Crustdata account usage', error) return false } } @@ -266,7 +264,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE } if (data.email) { - for (const email of data.email.split(',').filter(isValidEmail)) { + for (const email of data.email.split(',').filter(isEmail)) { normalized.identities.push({ type: MemberIdentityType.EMAIL, platform: this.platform, @@ -295,7 +293,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE data: IMemberEnrichmentDataCrustdata, normalized: IMemberEnrichmentDataNormalized, ): IMemberEnrichmentDataNormalized { - const employmentInformation = data.past_employers.concat(data.current_employers) + const employmentInformation = (data.past_employers || []).concat(data.current_employers || []) if (data.past_employers) { for (const workExperience of employmentInformation) { const identities = [] diff --git a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts index 8793565ebb..f5c4d45f3a 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/progai-linkedin-scraper/service.ts @@ -57,8 +57,6 @@ export default class EnrichmentServiceProgAILinkedinScraper hasEnrichableLinkedinInCache = true break } - - break } } @@ -86,11 +84,7 @@ export default class EnrichmentServiceProgAILinkedinScraper const existingProgaiCache = caches.find((c) => c.source === MemberEnrichmentSource.PROGAI) // we don't want to reinforce the cache with the same data, only save to cache // if a new profile is returned from progai - if ( - existingProgaiCache && - existingProgaiCache.data && - (existingProgaiCache.data as IMemberEnrichmentDataProgAI).id == data.id - ) { + if ((existingProgaiCache?.data as IMemberEnrichmentDataProgAI)?.id == data.id) { continue } profiles.push({ @@ -106,7 +100,9 @@ export default class EnrichmentServiceProgAILinkedinScraper return profiles.length > 0 ? profiles : null } - private async getDataUsingLinkedinHandle(handle: string): Promise { + private async getDataUsingLinkedinHandle( + handle: string, + ): Promise { const url = `${process.env['CROWD_ENRICHMENT_PROGAI_URL']}/get_profile` const config = { method: 'get', diff --git a/services/libs/common/src/strings.ts b/services/libs/common/src/strings.ts index 3c72f69be5..955497d01c 100644 --- a/services/libs/common/src/strings.ts +++ b/services/libs/common/src/strings.ts @@ -6,7 +6,3 @@ export const getCleanString = (value: string): string => { .toLowerCase() .trim() } - -export const isValidEmail = (email: string): boolean => { - return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email.trim()) -} diff --git a/services/libs/types/src/enums/members.ts b/services/libs/types/src/enums/members.ts index 31ac7805ba..47ec335bb7 100644 --- a/services/libs/types/src/enums/members.ts +++ b/services/libs/types/src/enums/members.ts @@ -156,6 +156,10 @@ export const MemberAttributes = { name: MemberAttributeName.EDUCATION, label: 'Education', }, + [MemberAttributeName.SCHOOLS]: { + name: MemberAttributeName.SCHOOLS, + label: 'Schools', + }, [MemberAttributeName.AWARDS]: { name: MemberAttributeName.AWARDS, label: 'Awards', From 5c44fe21cacd6ba8e52e7dc93f602e5693fb481a Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:26:31 +0100 Subject: [PATCH 04/13] add crustdata envs to required service envs on startup --- .../premium/members_enrichment_worker/src/main.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/main.ts b/services/apps/premium/members_enrichment_worker/src/main.ts index e255b09aa0..5897a2a4ec 100644 --- a/services/apps/premium/members_enrichment_worker/src/main.ts +++ b/services/apps/premium/members_enrichment_worker/src/main.ts @@ -1,8 +1,11 @@ -import { Config } from '@crowd/archetype-standard' -import { Options, ServiceWorker } from '@crowd/archetype-worker' -import { Edition } from '@crowd/types' +import { Config } from '@crowd/archetype-standard'; +import { Options, ServiceWorker } from '@crowd/archetype-worker'; +import { Edition } from '@crowd/types'; + + + +import { scheduleMembersEnrichment, scheduleMembersLFIDEnrichment } from './schedules'; -import { scheduleMembersEnrichment, scheduleMembersLFIDEnrichment } from './schedules' const config: Config = { envvars: [ @@ -12,6 +15,8 @@ const config: Config = { 'CROWD_ENRICHMENT_CLEARBIT_API_KEY', 'CROWD_ENRICHMENT_SERP_API_URL', 'CROWD_ENRICHMENT_SERP_API_KEY', + 'CROWD_ENRICHMENT_CRUSTDATA_URL', + 'CROWD_ENRICHMENT_CRUSTDATA_API_KEY', ], producer: { enabled: false, @@ -48,4 +53,4 @@ setImmediate(async () => { } await svc.start() -}) +}) \ No newline at end of file From 773b7503d08679dd3abff4757d2ab36498c0ad23 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:30:29 +0100 Subject: [PATCH 05/13] better employment normalization check --- .../src/sources/crustdata/service.ts | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index 11c50d91bd..2c28baa283 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -1,30 +1,22 @@ -import axios from 'axios' - -import { isEmail } from '@crowd/common' -import { Logger, LoggerBase } from '@crowd/logging' -import { - IMemberEnrichmentCache, - IMemberIdentity, - MemberAttributeName, - MemberEnrichmentSource, - MemberIdentityType, - OrganizationIdentityType, - OrganizationSource, - PlatformType, -} from '@crowd/types' - -import { findMemberEnrichmentCacheForAllSources } from '../../activities/enrichment' -import { EnrichmentSourceServiceFactory } from '../../factory' -import { - IEnrichmentService, - IEnrichmentSourceInput, - IMemberEnrichmentAttributeSettings, - IMemberEnrichmentData, - IMemberEnrichmentDataNormalized, -} from '../../types' -import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common' - -import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types' +import axios from 'axios'; + + + +import { isEmail } from '@crowd/common'; +import { Logger, LoggerBase } from '@crowd/logging'; +import { IMemberEnrichmentCache, IMemberIdentity, MemberAttributeName, MemberEnrichmentSource, MemberIdentityType, OrganizationIdentityType, OrganizationSource, PlatformType } from '@crowd/types'; + + + +import { findMemberEnrichmentCacheForAllSources } from '../../activities/enrichment'; +import { EnrichmentSourceServiceFactory } from '../../factory'; +import { IEnrichmentService, IEnrichmentSourceInput, IMemberEnrichmentAttributeSettings, IMemberEnrichmentData, IMemberEnrichmentDataNormalized } from '../../types'; +import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common'; + + + +import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types'; + export default class EnrichmentServiceCrustdata extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.CRUSTDATA @@ -294,7 +286,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE normalized: IMemberEnrichmentDataNormalized, ): IMemberEnrichmentDataNormalized { const employmentInformation = (data.past_employers || []).concat(data.current_employers || []) - if (data.past_employers) { + if (employmentInformation.length > 0) { for (const workExperience of employmentInformation) { const identities = [] @@ -321,4 +313,4 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE return normalized } -} +} \ No newline at end of file From 0442a2b0104c245798bc7f309be7409118584a53 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:39:35 +0100 Subject: [PATCH 06/13] formatting --- .../src/sources/crustdata/service.ts | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index 2c28baa283..be9dd5e884 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -1,22 +1,30 @@ -import axios from 'axios'; - - - -import { isEmail } from '@crowd/common'; -import { Logger, LoggerBase } from '@crowd/logging'; -import { IMemberEnrichmentCache, IMemberIdentity, MemberAttributeName, MemberEnrichmentSource, MemberIdentityType, OrganizationIdentityType, OrganizationSource, PlatformType } from '@crowd/types'; - - - -import { findMemberEnrichmentCacheForAllSources } from '../../activities/enrichment'; -import { EnrichmentSourceServiceFactory } from '../../factory'; -import { IEnrichmentService, IEnrichmentSourceInput, IMemberEnrichmentAttributeSettings, IMemberEnrichmentData, IMemberEnrichmentDataNormalized } from '../../types'; -import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common'; - - - -import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types'; - +import axios from 'axios' + +import { isEmail } from '@crowd/common' +import { Logger, LoggerBase } from '@crowd/logging' +import { + IMemberEnrichmentCache, + IMemberIdentity, + MemberAttributeName, + MemberEnrichmentSource, + MemberIdentityType, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { findMemberEnrichmentCacheForAllSources } from '../../activities/enrichment' +import { EnrichmentSourceServiceFactory } from '../../factory' +import { + IEnrichmentService, + IEnrichmentSourceInput, + IMemberEnrichmentAttributeSettings, + IMemberEnrichmentData, + IMemberEnrichmentDataNormalized, +} from '../../types' +import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common' + +import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types' export default class EnrichmentServiceCrustdata extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.CRUSTDATA @@ -313,4 +321,4 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE return normalized } -} \ No newline at end of file +} From 020052a1a9ce3f7160e9cb1891afde21ed844fb6 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:42:33 +0100 Subject: [PATCH 07/13] add initialization for normalizeEmployment for possible edge cases --- .../src/sources/crustdata/service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index be9dd5e884..f409c88cba 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -293,6 +293,10 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE data: IMemberEnrichmentDataCrustdata, normalized: IMemberEnrichmentDataNormalized, ): IMemberEnrichmentDataNormalized { + if (!normalized.memberOrganizations) { + normalized.memberOrganizations = [] + } + const employmentInformation = (data.past_employers || []).concat(data.current_employers || []) if (employmentInformation.length > 0) { for (const workExperience of employmentInformation) { From 9d7f09404488ccac63695e311de8374a0a524200 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:43:13 +0100 Subject: [PATCH 08/13] remove contributions from normalized object in crustdata --- .../members_enrichment_worker/src/sources/crustdata/service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index f409c88cba..484c56a85f 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -231,7 +231,6 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE let normalized: IMemberEnrichmentDataNormalized = { identities: [], attributes: {}, - contributions: [], memberOrganizations: [], reach: {}, } From a655e060a7874b837280bf38e9f0f1620f9ba4fa Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 11:43:47 +0100 Subject: [PATCH 09/13] formatting --- .../premium/members_enrichment_worker/src/main.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/main.ts b/services/apps/premium/members_enrichment_worker/src/main.ts index 5897a2a4ec..7603252158 100644 --- a/services/apps/premium/members_enrichment_worker/src/main.ts +++ b/services/apps/premium/members_enrichment_worker/src/main.ts @@ -1,11 +1,8 @@ -import { Config } from '@crowd/archetype-standard'; -import { Options, ServiceWorker } from '@crowd/archetype-worker'; -import { Edition } from '@crowd/types'; - - - -import { scheduleMembersEnrichment, scheduleMembersLFIDEnrichment } from './schedules'; +import { Config } from '@crowd/archetype-standard' +import { Options, ServiceWorker } from '@crowd/archetype-worker' +import { Edition } from '@crowd/types' +import { scheduleMembersEnrichment, scheduleMembersLFIDEnrichment } from './schedules' const config: Config = { envvars: [ @@ -53,4 +50,4 @@ setImmediate(async () => { } await svc.start() -}) \ No newline at end of file +}) From 3f717a3d183ee9f2d9e68855bc70d06bb3f80897 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 12:05:16 +0100 Subject: [PATCH 10/13] comments, crustdata getData better error handling --- .../src/sources/crustdata/service.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index 484c56a85f..bc3432f762 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -107,6 +107,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE const response: IMemberEnrichmentCrustdataRemainingCredits = (await axios(config)).data + // realtime linkedin enrichment costs 5 credits return response.credits > 5 } catch (error) { this.log.error('Error while checking Crustdata account usage', error) @@ -154,7 +155,11 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE const response: IMemberEnrichmentDataCrustdata[] = (await axios(config)).data - return response.length > 0 ? response[0] : null + if (response.length === 0 || response[0].error) { + return null + } + + return response[0] } private async findDistinctScrapableLinkedinIdentities( From e95edfcdda6e6976517757e7a9f69811cfbd9958 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 12:12:09 +0100 Subject: [PATCH 11/13] increase page size to 500 when getting enrichable members --- .../src/workflows/getMembersToEnrich.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts index 6124f83223..6439666ac5 100644 --- a/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts +++ b/services/apps/premium/members_enrichment_worker/src/workflows/getMembersToEnrich.ts @@ -18,7 +18,7 @@ const { getEnrichableMembers } = proxyActivities({ }) export async function getMembersToEnrich(args: IGetMembersForEnrichmentArgs): Promise { - const MEMBER_ENRICHMENT_PER_RUN = 100 + const MEMBER_ENRICHMENT_PER_RUN = 500 const afterCursor = args?.afterCursor || null const sources = [ MemberEnrichmentSource.PROGAI, From 033f52c08ee2e2afc0b87f5928afbb2006bc0bfe Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 12:41:31 +0100 Subject: [PATCH 12/13] better error handling --- .../src/sources/crustdata/service.ts | 17 ++++++++++++++--- .../src/sources/crustdata/types.ts | 11 +++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index bc3432f762..ec3296cb22 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -24,7 +24,12 @@ import { } from '../../types' import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common' -import { IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata } from './types' +import { + IMemberEnrichmentCrustdataAPIErrorResponse, + IMemberEnrichmentCrustdataAPIResponse, + IMemberEnrichmentCrustdataRemainingCredits, + IMemberEnrichmentDataCrustdata, +} from './types' export default class EnrichmentServiceCrustdata extends LoggerBase implements IEnrichmentService { public source: MemberEnrichmentSource = MemberEnrichmentSource.CRUSTDATA @@ -153,15 +158,21 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE }, } - const response: IMemberEnrichmentDataCrustdata[] = (await axios(config)).data + const response: IMemberEnrichmentCrustdataAPIResponse[] = (await axios(config)).data - if (response.length === 0 || response[0].error) { + if (response.length === 0 || this.isErrorResponse(response[0])) { return null } return response[0] } + private isErrorResponse( + response: IMemberEnrichmentCrustdataAPIResponse, + ): response is IMemberEnrichmentCrustdataAPIErrorResponse { + return (response as IMemberEnrichmentCrustdataAPIErrorResponse).error !== undefined + } + private async findDistinctScrapableLinkedinIdentities( input: IEnrichmentSourceInput, caches: IMemberEnrichmentCache[], diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts index a88c97aa5d..6fe2266261 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/types.ts @@ -37,6 +37,17 @@ export interface IMemberEnrichmentDataCrustdata { metadata: IMemberEnrichmentLinkedinScraperMetadata } +export type IMemberEnrichmentCrustdataAPIResponse = + | IMemberEnrichmentDataCrustdata + | IMemberEnrichmentCrustdataAPIErrorResponse + +export interface IMemberEnrichmentCrustdataAPIErrorResponse { + error: string + linkedin_profile_url: string + last_tried_linkedin_enrichment_date: string + did_last_linkedin_enrichment_succeed: boolean +} + export interface IMemberEnrichmentCrustdataRemainingCredits { credits: number } From bc48cb1e131834c85221ca5f2d56f52d604d6fb1 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 7 Nov 2024 12:47:20 +0100 Subject: [PATCH 13/13] switch to realtime processing before deploy --- .../members_enrichment_worker/src/sources/crustdata/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts index ec3296cb22..6e2e20808e 100644 --- a/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/premium/members_enrichment_worker/src/sources/crustdata/service.ts @@ -151,7 +151,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE url, params: { linkedin_profile_url: `https://linkedin.com/in/${handle}`, - enrich_realtime: false, + enrich_realtime: true, }, headers: { Authorization: `Token ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`,