Improve websiteNormalizer with public suffix list (#1413)

Author: skwowet

services/libs/common/package-lock.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@crowd/logging": "file:../logging",
     "@crowd/types": "file:../types",
+    "psl": "^1.9.0",
     "uuid": "^9.0.0"
   }
 }

services/libs/common/package.json

@@ -1,15 +1,13 @@
-export const websiteNormalizer = (website: string): string => {
-  // Prepends https:// to make valid URL
-  const completeUrl = website.includes('://') ? website : 'https://' + website
+import { parse, isValid } from 'psl'
 
-  const url = new URL(completeUrl)
-  const hostname = url.hostname
+export const websiteNormalizer = (website: string): string => {
+  // remove http:// or https://
+  const cleanURL = website.replace(/(^\w+:|^)\/\//, '')
+  const parsed = parse(cleanURL)
 
-  const parts = hostname.split('.')
-  // Ignore subdomains, return only domain and TLD
-  if (parts.length > 2) {
-    return parts.slice(-2).join('.')
+  if (!isValid(cleanURL)) {
+    return null
   }
 
-  return hostname
+  return parsed.domain
 }