Merge branch 'main' into crowd-linux

Author: skwowet

backend/src/services/organizationService.ts

@@ -1,4 +1,5 @@
 import { LoggerBase } from '@crowd/logging'
+import { websiteNormalizer } from '@crowd/common'
 import { CLEARBIT_CONFIG, IS_TEST_ENV } from '../conf'
 import MemberRepository from '../database/repositories/memberRepository'
 import organizationCacheRepository from '../database/repositories/organizationCacheRepository'
@@ -43,6 +44,11 @@ export default class OrganizationService extends LoggerBase {
         transaction,
       })
 
+      // Normalize the website URL if it exists
+      if (data.website) {
+        data.website = websiteNormalizer(data.website)
+      }
+
       // if cache exists, merge current data with cache data
       // if it doesn't exist, create it from incoming data
       if (cache) {
@@ -146,6 +152,11 @@ export default class OrganizationService extends LoggerBase {
         })
       }
 
+      // Normalize the website URL if it exists
+      if (data.website) {
+        data.website = websiteNormalizer(data.website)
+      }
+
       const record = await OrganizationRepository.update(id, data, {
         ...this.options,
         transaction,

services/apps/data_sink_worker/src/service/organization.service.ts

@@ -6,6 +6,7 @@ import { OrganizationRepository } from '@/repo/organization.repo'
 import { DbStore } from '@crowd/database'
 import { Logger, LoggerBase, getChildLogger } from '@crowd/logging'
 import { IOrganization, IOrganizationSocial, PlatformType } from '@crowd/types'
+import { websiteNormalizer } from '@crowd/common'
 
 export class OrganizationService extends LoggerBase {
   private readonly repo: OrganizationRepository
@@ -23,6 +24,11 @@ export class OrganizationService extends LoggerBase {
   ): Promise<string> {
     data = this.normalizeSocialFields(data)
 
+    // Normalize the website URL if it exists
+    if (data.website) {
+      data.website = websiteNormalizer(data.website)
+    }
+
     // find from cache by name
     let cached = await this.repo.findCacheByName(data.name)
 

services/libs/common/package-lock.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@crowd/logging": "file:../logging",
     "@crowd/types": "file:../types",
+    "psl": "^1.9.0",
     "uuid": "^9.0.0"
   }
 }

services/libs/common/package.json

@@ -11,3 +11,4 @@ export * from './requestThrottler'
 export * from './rawQueryParser'
 export * from './byteLength'
 export * from './http'
+export * from './websiteNormalizer'

services/libs/common/src/index.ts

@@ -0,0 +1,13 @@
+import { parse, isValid } from 'psl'
+
+export const websiteNormalizer = (website: string): string => {
+  // remove http:// or https://
+  const cleanURL = website.replace(/(^\w+:|^)\/\//, '')
+  const parsed = parse(cleanURL)
+
+  if (!isValid(cleanURL)) {
+    return null
+  }
+
+  return parsed.domain
+}