Building a Media Monitoring Tool: Mentions Across News Sites

Media monitoring, tracking brand mentions, competitor coverage, and industry news, is a service companies pay thousands for monthly. Here’s how to build your own with Python and web scraping.

What We’ll Build

A tool that:

Monitors multiple news sources for keyword mentions
Extracts article metadata (title, date, author, snippet)
Deduplicates across sources
Sends alerts for new mentions

Setup

pip install requests beautifulsoup4 pandas feedparser newspaper3k

RSS Feed Monitor

The fastest approach starts with RSS feeds, as many news sites still publish them:

import feedparser
from datetime import datetime

class RSSMonitor:
    def __init__(self, keywords):
        self.keywords = [k.lower() for k in keywords]
        self.seen_urls = set()

    def check_feed(self, feed_url, source_name):
        """Check an RSS feed for keyword mentions."""
        feed = feedparser.parse(feed_url)
        mentions = []

        for entry in feed.entries:
            title = entry.get("title", "")
            summary = entry.get("summary", "")
            link = entry.get("link", "")

            if link in self.seen_urls:
                continue

            text = f"{title} {summary}".lower()
            matched = [k for k in self.keywords if k in text]

            if matched:
                self.seen_urls.add(link)
                mentions.append({
                    "source": source_name,
                    "title": title,
                    "url": link,
                    "summary": summary[:300],
                    "keywords_matched": matched,
                    "published": entry.get("published", ""),
                    "found_at": datetime.now().isoformat()
                })

        return mentions

Web Scraping for Sites Without RSS

import requests
from bs4 import BeautifulSoup

SCRAPER_API_KEY = "YOUR_KEY"

class WebMonitor:
    def __init__(self, keywords):
        self.keywords = [k.lower() for k in keywords]
        self.seen_urls = set()

    def scrape_news_site(self, base_url, source_name, article_selector, title_selector):
        """Scrape a news site for articles matching keywords."""
        api_url = (
            f"http://api.scraperapi.com?api_key={SCRAPER_API_KEY}"
            f"&url={base_url}"
        )
        response = requests.get(api_url, timeout=30)
        soup = BeautifulSoup(response.text, "html.parser")

        mentions = []
        for article in soup.select(article_selector):
            title_el = article.select_one(title_selector)
            link_el = article.select_one("a[href]")

            if not title_el or not link_el:
                continue

            title = title_el.text.strip()
            url = link_el["href"]
            if not url.startswith("http"):
                url = base_url.rstrip("/") + url

            if url in self.seen_urls:
                continue

            matched = [k for k in self.keywords if k in title.lower()]

            if matched:
                self.seen_urls.add(url)
                mentions.append({
                    "source": source_name,
                    "title": title,
                    "url": url,
                    "keywords_matched": matched,
                    "found_at": datetime.now().isoformat()
                })

        return mentions

Full Article Extraction

from newspaper import Article

def extract_full_article(url):
    """Extract full article text and metadata."""
    try:
        article = Article(url)
        article.download()
        article.parse()

        return {
            "title": article.title,
            "authors": article.authors,
            "publish_date": str(article.publish_date) if article.publish_date else "",
            "text": article.text,
            "top_image": article.top_image,
            "word_count": len(article.text.split())
        }
    except Exception as e:
        return {"error": str(e)}

The Monitoring Pipeline

import time
import json

NEWS_FEEDS = [
    ("TechCrunch", "https://techcrunch.com/feed/"),
    ("The Verge", "https://www.theverge.com/rss/index.xml"),
    ("Ars Technica", "https://feeds.arstechnica.com/arstechnica/index"),
]

NEWS_SITES = [
    ("Reuters Tech", "https://www.reuters.com/technology/", "article", "h3"),
]

def run_monitoring(keywords, interval_minutes=30):
    rss = RSSMonitor(keywords)
    web = WebMonitor(keywords)
    all_mentions = []

    while True:
        print(f"n[{datetime.now()}] Checking sources...")

        for name, url in NEWS_FEEDS:
            mentions = rss.check_feed(url, name)
            all_mentions.extend(mentions)
            for m in mentions:
                print(f"  NEW: [{m['source']}] {m['title']}")

        for name, url, art_sel, title_sel in NEWS_SITES:
            try:
                mentions = web.scrape_news_site(url, name, art_sel, title_sel)
                all_mentions.extend(mentions)
                for m in mentions:
                    print(f"  NEW: [{m['source']}] {m['title']}")
            except Exception as e:
                print(f"  Error with {name}: {e}")
            time.sleep(3)

        with open("mentions.json", "w") as f:
            json.dump(all_mentions, f, indent=2)

        print(f"Total mentions: {len(all_mentions)}")
        time.sleep(interval_minutes * 60)

# Run it
run_monitoring(["openai", "anthropic", "google ai", "llm"])

Scaling Considerations

For production media monitoring, ScraperAPI handles proxy rotation across news sites reliably. ThorData residential proxies help with geo-restricted content. Track your scraper health metrics with ScrapeOps.

Conclusion

A custom media monitoring tool gives you real-time awareness without expensive SaaS subscriptions. Start with RSS feeds for speed, add web scraping for coverage, and build alerts from there.