Detect a website's tech stack for B2B lead research (Python + Node.js)

"""Guess a site's CMS, shop system, CDN and email provider with the standard library only. Usage: python techstack_stdlib.py shopware.com wordpress.org allbirds.com """ import json import re import sys import urllib.request UA = {"User-Agent": "Mozilla/5.0 (techstack-demo; +https://example.com/contact)"} # (technology, where to look, regex) - a tiny subset of real fingerprint sets RULES = [ ("WordPress", "html", r"/wp-content/|/wp-includes/|<meta[^>]+generator[^>]+WordPress"), ("WooCommerce", "html", r"woocommerce"), ("Shopify", "html", r"cdn\.shopify\.com|Shopify\.theme"), ("Shopware", "html", r"/bundles/storefront/|sw-plugin|Shopware"), ("Contentful", "html", r"ctfassets\.net"), ("Next.js", "html", r"/_next/static/|__NEXT_DATA__"), ("Google Tag Manager", "html", r"googletagmanager\.com/gtm\.js"), ("Cloudflare", "headers", r"^server: cloudflare|^cf-ray:"), ("Fastly", "headers", r"^x-served-by: cache-|^fastly-"), ("Amazon CloudFront", "headers", r"^x-amz-cf-id:|^via: .*cloudfront"), ("nginx", "headers", r"^server: nginx"), ] MX = {"google.com": "Google Workspace", "googlemail.com": "Google Workspace", "outlook.com": "Microsoft 365", "pphosted.com": "Proofpoint", "mimecast.com": "Mimecast", "zoho": "Zoho Mail"} def fetch(domain): req = urllib.request.Request(f"https://{domain}/", headers=UA) with urllib.request.urlopen(req, timeout=20) as r: headers = "\n".join(f"{k.lower()}: {v.lower()}" for k, v in r.headers.items()) return headers, r.read(3_000_000).decode("utf-8", "replace") def mail_provider(domain): url = f"https://dns.google/resolve?name={domain}&type=MX" # DNS over HTTPS, no extra library  answers = json.load(urllib.request.urlopen(url, timeout=10)).get("Answer", []) hosts = [a["data"].split()[-1].rstrip(".").lower() for a in answers if a.get("type") == 15] for h in hosts: for key, name in MX.items(): if key in h: return name return "self-hosted / other" if hosts else "none" for domain in sys.argv[1:]: try: headers, html = fetch(domain) except Exception as exc: # noqa: BLE001 - one bad site must not stop the list  print(f"{domain:<16} ERROR {exc}") continue found = [t for t, where, rx in RULES if re.search(rx, headers if where == "headers" else html, re.I | re.M)] print(f"{domain:<16} {', '.join(found) or '-'} | mail: {mail_provider(domain)}") 

$ python techstack_stdlib.py shopware.com wordpress.org allbirds.com shopware.com Shopware, Contentful, Fastly, nginx | mail: Microsoft 365 wordpress.org WordPress, WooCommerce, Google Tag Manager, nginx | mail: self-hosted / other allbirds.com Shopify, Google Tag Manager, Cloudflare | mail: Microsoft 365 

"""Detect the tech stack of a list of websites and save CMS, shop system and more to techstack.csv. Actor: https://apify.com/paperlinenstudio/tech-stack-detector Usage: APIFY_TOKEN=... python tech_stack_leads.py shopware.com wordpress.org shopify.com Lead filter: ONLY=Shopify,WooCommerce python tech_stack_leads.py ... keeps only matching sites (others are not charged). """ import csv import os import sys from apify_client import ApifyClient domains = sys.argv[1:] or ["shopware.com", "wordpress.org", "shopify.com"] ONLY = [t.strip() for t in os.environ.get("ONLY", "").split(",") if t.strip()] client = ApifyClient(os.environ["APIFY_TOKEN"]) run = client.actor("paperlinenstudio/tech-stack-detector").call( run_input={"urls": domains, "filterTechnologies": ONLY, "includeEvidence": False} ) rows = list(client.dataset(run.default_dataset_id).iterate_items()) cols = ["domain", "cms", "ecommerce", "jsFrameworks", "analytics", "consentManagers", "cdn", "hosting", "emailProvider", "technologyCount", "error"] with open("techstack.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(cols) for r in rows: w.writerow([", ".join(v) if isinstance(v, list) else v for v in (r.get(c) for c in cols)]) for r in rows: show = lambda v: (", ".join(v) if isinstance(v, list) else v) or "-" # noqa: E731  print(f"{r['domain']:<20} cms={show(r.get('cms'))} shop={show(r.get('ecommerce'))} cdn={show(r.get('cdn'))} mail={show(r.get('emailProvider'))}") print(f"Saved {len(rows)} rows to techstack.csv") 

$ ONLY=Shopify,WooCommerce python tech_stack_leads.py allbirds.com gymshark.com wordpress.org shopware.com woocommerce.com woocommerce.com cms=WordPress shop=WooCommerce cdn=- mail=Self-hosted / other gymshark.com cms=- shop=Shopify cdn=Cloudflare mail=Proofpoint allbirds.com cms=- shop=Shopify cdn=Cloudflare mail=Microsoft 365 Saved 3 rows to techstack.csv 

// Detect the tech stack of a list of websites and save CMS, shop system and more to techstack.csv. // Usage: APIFY_TOKEN=... node tech_stack_leads.mjs shopware.com wordpress.org shopify.com // Lead filter: ONLY=Shopify,WooCommerce node tech_stack_leads.mjs ... keeps only matching sites (others are not charged). import { writeFileSync } from 'node:fs'; import { ApifyClient } from 'apify-client'; const domains = process.argv.slice(2).length ? process.argv.slice(2) : ['shopware.com', 'wordpress.org', 'shopify.com']; const ONLY = (process.env.ONLY ?? '').split(',').map((t) => t.trim()).filter(Boolean); const client = new ApifyClient({ token: process.env.APIFY_TOKEN }); const run = await client.actor('paperlinenstudio/tech-stack-detector').call({ urls: domains, filterTechnologies: ONLY, includeEvidence: false }); const { items } = await client.dataset(run.defaultDatasetId).listItems(); const cols = ['domain', 'cms', 'ecommerce', 'jsFrameworks', 'analytics', 'consentManagers', 'cdn', 'hosting', 'emailProvider', 'technologyCount', 'error']; const cell = (v) => { const s = Array.isArray(v) ? v.join(', ') : (v ?? ''); return `"${String(s).replaceAll('"', '""')}"`; }; writeFileSync('techstack.csv', [cols.join(','), ...items.map((r) => cols.map((c) => cell(r[c])).join(','))].join('\n') + '\n'); const show = (v) => (Array.isArray(v) ? v.join(', ') : v) || '-'; for (const r of items) console.log(`${r.domain.padEnd(20)} cms=${show(r.cms)} shop=${show(r.ecommerce)} cdn=${show(r.cdn)} mail=${show(r.emailProvider)}`); console.log(`Saved ${items.length} rows to techstack.csv`); 

$ node tech_stack_leads.mjs wordpress.org cms=WordPress shop=- cdn=- mail=Self-hosted / other shopify.com cms=- shop=Shopify cdn=Cloudflare mail=Google Workspace shopware.com cms=Contentful shop=- cdn=Fastly mail=Microsoft 365 Saved 3 rows to techstack.csv 

{ "url": "https://shopware.com/", "domain": "shopware.com", "finalUrl": "https://www.shopware.com/en/", "statusCode": 200, "redirects": 2, "https": true, "title": "The ecommerce platform to power your online business | Shopware", "language": "en", "generator": null, "cms": "Contentful", "ecommerce": null, "jsFrameworks": [ "Nuxt", "Vue.js" ], "analytics": [], "tagManagers": [ "Google Tag Manager" ], "advertising": [], "consentManagers": [ "Usercentrics" ], "cdn": [ "Fastly" ], "hosting": [], "webServers": [ "Nginx" ], "backend": [], "liveChat": [], "marketingAutomation": [], "payments": [], "emailProvider": "Microsoft 365", "mxRecords": [ "shopware-com.mail.protection.outlook.com" ], "technologyCount": 14, "technologyNames": [ "Fastly", "Contentful", "Varnish", "Usercentrics", "Nuxt", "Vue.js", "PWA (Web App Manifest)", "Open Graph", "Schema.org JSON-LD", "Friendly Captcha", "Content Security Policy", "HSTS", "Google Tag Manager",Source：https://dev.to/paperandlinenstudio/detect-a-websites-tech-stack-for-b2b-lead-research-python-nodejs-337j
My favorites