diff --git a/backend/src/database/migrations/U1775064222__addCommitteesActivityTypes.sql b/backend/src/database/migrations/U1775064222__addCommitteesActivityTypes.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1775064222__addCommitteesActivityTypes.sql b/backend/src/database/migrations/V1775064222__addCommitteesActivityTypes.sql new file mode 100644 index 0000000000..7e924d24c7 --- /dev/null +++ b/backend/src/database/migrations/V1775064222__addCommitteesActivityTypes.sql @@ -0,0 +1,3 @@ +INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description, "label") VALUES +('added-to-committee', 'committees', false, false, 'Member is added to a committee', 'Added to committee'), +('removed-from-committee', 'committees', false, false, 'Member is removed from a committee', 'Removed from committee'); diff --git a/services/apps/snowflake_connectors/src/integrations/committees/committees/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/committees/committees/buildSourceQuery.ts new file mode 100644 index 0000000000..c21de53832 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/committees/committees/buildSourceQuery.ts @@ -0,0 +1,117 @@ +import { IS_PROD_ENV } from '@crowd/common' + +// Main: FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C +// Joins: +// - ANALYTICS.SILVER_DIM.COMMITTEE (committee metadata + project slug) +// - ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS (segment resolution) +// - ANALYTICS.SILVER_DIM.USERS (member identity: email, lf_username, name) +// - ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.USERS (actor name + email) +// - ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS (org data) + +const CDP_MATCHED_SEGMENTS = ` + cdp_matched_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s + WHERE s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + )` + +const ORG_ACCOUNTS = ` + org_accounts AS ( + SELECT account_id, account_name, website, domain_aliases + FROM ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS + WHERE website IS NOT NULL + )` + +export const buildSourceQuery = (sinceTimestamp?: string): string => { + let select = ` + SELECT + c.SFID, + c._FIVETRAN_DELETED AS FIVETRAN_DELETED, + c.CONTACTEMAIL__C, + c.CREATEDBYID, + c.COLLABORATION_NAME__C, + c.ACCOUNT__C, + c.ROLE__C, + c.CREATEDDATE::TIMESTAMP_NTZ AS CREATEDDATE, + c.LASTMODIFIEDDATE::TIMESTAMP_NTZ AS LASTMODIFIEDDATE, + c._FIVETRAN_SYNCED::TIMESTAMP_NTZ AS FIVETRAN_SYNCED, + cm.COMMITTEE_ID, + cm.COMMITTEE_NAME, + cm.PROJECT_ID, + cm.PROJECT_NAME, + cm.PROJECT_SLUG, + su.EMAIL AS SU_EMAIL, + su.LF_USERNAME, + su.PRIMARY_SOURCE_USER_ID, + su.FIRST_NAME AS SU_FIRST_NAME, + su.LAST_NAME AS SU_LAST_NAME, + su.FULL_NAME AS SU_FULL_NAME, + bu.FIRST_NAME AS BU_FIRST_NAME, + bu.LAST_NAME AS BU_LAST_NAME, + bu.EMAIL AS BU_EMAIL, + org.account_name AS ACCOUNT_NAME, + org.website AS ORG_WEBSITE, + org.domain_aliases AS ORG_DOMAIN_ALIASES + FROM FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C c + JOIN ANALYTICS.SILVER_DIM.COMMITTEE cm + ON c.COLLABORATION_NAME__C = cm.COMMITTEE_ID + INNER JOIN cdp_matched_segments cms + ON cms.slug = cm.PROJECT_SLUG + AND cms.sourceId = cm.PROJECT_ID + LEFT JOIN ANALYTICS.SILVER_DIM.USERS su + ON LOWER(c.CONTACTEMAIL__C) = LOWER(su.EMAIL) + LEFT JOIN ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.USERS bu + ON c.CREATEDBYID = bu.USER_ID + LEFT JOIN org_accounts org + ON c.ACCOUNT__C = org.account_id + WHERE c.LASTMODIFIEDDATE IS NOT NULL` + + // Limit to a single project in non-prod to avoid exporting all project data + if (!IS_PROD_ENV) { + select += ` AND cm.PROJECT_SLUG = 'cncf'` + } + + const dedup = ` + QUALIFY ROW_NUMBER() OVER (PARTITION BY c.SFID ORDER BY org.website DESC) = 1` + + if (!sinceTimestamp) { + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS} + ${select} + ${dedup}`.trim() + } + + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS}, + new_cdp_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s + WHERE s.CREATED_TS >= '${sinceTimestamp}' + AND s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + ) + + -- Updated committee memberships since last export + ${select} + AND c.LASTMODIFIEDDATE > '${sinceTimestamp}' + ${dedup} + + UNION + + -- All committee memberships in newly created segments + ${select} + AND EXISTS ( + SELECT 1 FROM new_cdp_segments ncs + WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId + ) + ${dedup}`.trim() +} diff --git a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts new file mode 100644 index 0000000000..24f772de55 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts @@ -0,0 +1,174 @@ +import { COMMITTEES_GRID, CommitteesActivityType } from '@crowd/integrations' +import { getServiceChildLogger } from '@crowd/logging' +import { + IActivityData, + IOrganizationIdentity, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' + +const log = getServiceChildLogger('committeesCommitteesTransformer') + +export class CommitteesCommitteesTransformer extends TransformerBase { + readonly platform = PlatformType.COMMITTEES + + transformRow(row: Record): TransformedActivity | null { + const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null + if (!email) { + log.warn( + { sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C }, + 'Skipping row: missing email', + ) + return null + } + + const committeeId = (row.COMMITTEE_ID as string).trim() + const fivetranDeleted = row.FIVETRAN_DELETED as boolean + const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null + const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null + const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null + const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null + + const displayName = + suFullName || + (suFirstName && suLastName ? `${suFirstName} ${suLastName}` : suFirstName || suLastName) || + email.split('@')[0] + + const type = fivetranDeleted + ? CommitteesActivityType.REMOVED_FROM_COMMITTEE + : CommitteesActivityType.ADDED_TO_COMMITTEE + + const sourceId = (row.PRIMARY_SOURCE_USER_ID as string | null)?.trim() || undefined + const identities = this.buildMemberIdentities({ + email, + platformUsername: null, + sourceId, + lfUsername, + }) + + const activityTimestamp = + type === CommitteesActivityType.ADDED_TO_COMMITTEE + ? (row.CREATEDDATE as string | null) || null + : (row.FIVETRAN_SYNCED as string | null) || null + + const committeeName = (row.COMMITTEE_NAME as string | null) || null + + const activity: IActivityData = { + type, + platform: PlatformType.COMMITTEES, + timestamp: activityTimestamp, + score: COMMITTEES_GRID[type].score, + sourceId: `${committeeId}-${row.SFID}`, + sourceParentId: null, + channel: committeeName, + member: { + displayName, + identities, + organizations: this.buildOrganizations(row), + }, + attributes: { + committeeId: (row.COLLABORATION_NAME__C as string | null) || null, + committeeName, + role: (row.ROLE__C as string | null) || null, + projectId: (row.PROJECT_ID as string | null) || null, + projectName: (row.PROJECT_NAME as string | null) || null, + organizationId: (row.ACCOUNT__C as string | null) || null, + organizationName: (row.ACCOUNT_NAME as string | null) || null, + member: { + userId: (row.PRIMARY_SOURCE_USER_ID as string | null) || null, + firstName: (row.SU_FIRST_NAME as string | null) || null, + lastName: (row.SU_LAST_NAME as string | null) || null, + email, + }, + actor: { + userId: (row.CREATEDBYID as string | null) || null, + firstName: (row.BU_FIRST_NAME as string | null) || null, + lastName: (row.BU_LAST_NAME as string | null) || null, + email: (row.BU_EMAIL as string | null) || null, + }, + activityDate: activityTimestamp, + }, + } + + const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null + const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null + + if (!segmentSlug || !segmentSourceId) { + log.warn( + { sfid: row.SFID, committeeId, segmentSlug, segmentSourceId }, + 'Skipping row: missing segment slug or sourceId', + ) + return null + } + + return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } + } + + private buildOrganizations( + row: Record, + ): IActivityData['member']['organizations'] { + const website = (row.ORG_WEBSITE as string | null)?.trim() || null + const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null + + if (!website && !domainAliases) { + return undefined + } + + const displayName = (row.ACCOUNT_NAME as string | null)?.trim() || website + + if (this.isIndividualNoAccount(displayName)) { + return [ + { + displayName, + source: OrganizationSource.COMMITTEES, + identities: website + ? [ + { + platform: PlatformType.COMMITTEES, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }, + ] + : [], + }, + ] + } + + const identities: IOrganizationIdentity[] = [] + + if (website) { + identities.push({ + platform: PlatformType.COMMITTEES, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }) + } + + if (domainAliases) { + for (const alias of domainAliases.split(',')) { + const trimmed = alias.trim() + if (trimmed) { + identities.push({ + platform: PlatformType.COMMITTEES, + value: trimmed, + type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, + verified: true, + }) + } + } + } + + return [ + { + displayName, + source: OrganizationSource.COMMITTEES, + identities, + }, + ] + } +} diff --git a/services/apps/snowflake_connectors/src/integrations/index.ts b/services/apps/snowflake_connectors/src/integrations/index.ts index 7adc90bec0..f04f6674f7 100644 --- a/services/apps/snowflake_connectors/src/integrations/index.ts +++ b/services/apps/snowflake_connectors/src/integrations/index.ts @@ -6,6 +6,8 @@ */ import { PlatformType } from '@crowd/types' +import { buildSourceQuery as committeesCommitteesBuildQuery } from './committees/committees/buildSourceQuery' +import { CommitteesCommitteesTransformer } from './committees/committees/transformer' import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registrations/buildSourceQuery' import { CventTransformer } from './cvent/event-registrations/transformer' import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery' @@ -20,6 +22,15 @@ export type { BuildSourceQuery, DataSource, PlatformDefinition } from './types' export { DataSourceName } from './types' const supported: Partial> = { + [PlatformType.COMMITTEES]: { + sources: [ + { + name: DataSourceName.COMMITTEES_COMMITTEES, + buildSourceQuery: committeesCommitteesBuildQuery, + transformer: new CommitteesCommitteesTransformer(), + }, + ], + }, [PlatformType.CVENT]: { sources: [ { diff --git a/services/apps/snowflake_connectors/src/integrations/types.ts b/services/apps/snowflake_connectors/src/integrations/types.ts index 36377a4f4f..6a5d6e85f4 100644 --- a/services/apps/snowflake_connectors/src/integrations/types.ts +++ b/services/apps/snowflake_connectors/src/integrations/types.ts @@ -8,6 +8,7 @@ export enum DataSourceName { TNC_ENROLLMENTS = 'enrollments', TNC_CERTIFICATES = 'certificates', TNC_COURSES = 'courses', + COMMITTEES_COMMITTEES = 'committees', } export interface DataSource { diff --git a/services/libs/data-access-layer/src/organizations/attributesConfig.ts b/services/libs/data-access-layer/src/organizations/attributesConfig.ts index c8d0414c3c..64399bb913 100644 --- a/services/libs/data-access-layer/src/organizations/attributesConfig.ts +++ b/services/libs/data-access-layer/src/organizations/attributesConfig.ts @@ -233,6 +233,7 @@ export const ORG_DB_ATTRIBUTE_SOURCE_PRIORITY = [ OrganizationAttributeSource.ENRICHMENT_PEOPLEDATALABS, OrganizationAttributeSource.CVENT, OrganizationAttributeSource.TNC, + OrganizationAttributeSource.COMMITTEES, // legacy - keeping this for backward compatibility OrganizationAttributeSource.ENRICHMENT, OrganizationAttributeSource.GITHUB, diff --git a/services/libs/integrations/src/integrations/committees/types.ts b/services/libs/integrations/src/integrations/committees/types.ts new file mode 100644 index 0000000000..9aaf32ab08 --- /dev/null +++ b/services/libs/integrations/src/integrations/committees/types.ts @@ -0,0 +1,11 @@ +import { IActivityScoringGrid } from '@crowd/types' + +export enum CommitteesActivityType { + ADDED_TO_COMMITTEE = 'added-to-committee', + REMOVED_FROM_COMMITTEE = 'removed-from-committee', +} + +export const COMMITTEES_GRID: Record = { + [CommitteesActivityType.ADDED_TO_COMMITTEE]: { score: 1 }, + [CommitteesActivityType.REMOVED_FROM_COMMITTEE]: { score: 1 }, +} diff --git a/services/libs/integrations/src/integrations/index.ts b/services/libs/integrations/src/integrations/index.ts index 3db691cd2b..d1a9f460b6 100644 --- a/services/libs/integrations/src/integrations/index.ts +++ b/services/libs/integrations/src/integrations/index.ts @@ -52,4 +52,6 @@ export * from './cvent/types' export * from './tnc/types' +export * from './committees/types' + export * from './activityDisplayService' diff --git a/services/libs/types/src/enums/organizations.ts b/services/libs/types/src/enums/organizations.ts index 3dc62f28dc..b209c3582f 100644 --- a/services/libs/types/src/enums/organizations.ts +++ b/services/libs/types/src/enums/organizations.ts @@ -14,6 +14,7 @@ export enum OrganizationSource { UI = 'ui', CVENT = 'cvent', TNC = 'tnc', + COMMITTEES = 'committees', } export enum OrganizationMergeSuggestionType { @@ -42,6 +43,7 @@ export enum OrganizationAttributeSource { ENRICHMENT_PEOPLEDATALABS = 'enrichment-peopledatalabs', CVENT = 'cvent', TNC = 'tnc', + COMMITTEES = 'committees', // legacy - keeping this for backward compatibility ENRICHMENT = 'enrichment', GITHUB = 'github', diff --git a/services/libs/types/src/enums/platforms.ts b/services/libs/types/src/enums/platforms.ts index 5090f17ad5..ad19e1f328 100644 --- a/services/libs/types/src/enums/platforms.ts +++ b/services/libs/types/src/enums/platforms.ts @@ -20,6 +20,7 @@ export enum PlatformType { GIT = 'git', CRUNCHBASE = 'crunchbase', GROUPSIO = 'groupsio', + COMMITTEES = 'committees', CONFLUENCE = 'confluence', GERRIT = 'gerrit', JIRA = 'jira',