[RFR] remove crimson hexagon references

.github/workflows/build.yml

@@ -173,7 +173,6 @@ jobs:
           # Map secrets to environment variables
           MC_BRANDWATCH_PASSWORD: ${{ secrets.MC_BRANDWATCH_PASSWORD }}
           MC_BRANDWATCH_USER: ${{ secrets.MC_BRANDWATCH_USER }}
-          MC_CRIMSON_HEXAGON_API_KEY: ${{ secrets.MC_CRIMSON_HEXAGON_API_KEY }}
           MC_DOWNLOADS_AMAZON_S3_ACCESS_KEY_ID: ${{ secrets.MC_DOWNLOADS_AMAZON_S3_ACCESS_KEY_ID }}
           MC_DOWNLOADS_AMAZON_S3_BUCKET_NAME: ${{ secrets.MC_DOWNLOADS_AMAZON_S3_BUCKET_NAME }}
           MC_DOWNLOADS_AMAZON_S3_DIRECTORY_NAME: ${{ secrets.MC_DOWNLOADS_AMAZON_S3_DIRECTORY_NAME }}

apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py

@@ -34,21 +34,21 @@ def test_get_feed_url_from_google_podcasts_url():
     assert _get_feed_url_from_google_podcasts_url('http://www.example.com/') == 'http://www.example.com/'
     assert _get_feed_url_from_google_podcasts_url('totally not an URL') == 'totally not an URL'
 
-    npr_feed_url = 'https://feeds.npr.org/381444908/podcast.xml'
+    ft_feed_url = 'https://rss.acast.com/ftnewsbriefing'
 
     # Test with URL pointing to a show's homepage (not invidual episode)
 
-    npr_google_show_url = (
-        'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM4MTQ0NDkwOC9wb2RjYXN0LnhtbA?sa=X'
-        '&ved=2ahUKEwjKm6fimbjuAhWMjoQIHUrSCW0Qjs4CKAl6BAgBEH4'
+    ft_google_show_url = (
+        'https://podcasts.google.com/feed/aHR0cHM6Ly9yc3MuYWNhc3QuY29tL2Z0bmV3c2JyaWVmaW5n?sa=X'
+        '&ved=0CH4Qjs4CKARqFwoTCIjZ5ZTNwvECFQAAAAAdAAAAABAL'
     )
 
-    assert _get_feed_url_from_google_podcasts_url(npr_google_show_url) == npr_feed_url
+    assert _get_feed_url_from_google_podcasts_url(ft_google_show_url) == ft_feed_url
 
     # Test with URL that points to a specific episode
-    npr_google_ep_url = (
-        'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM4MTQ0NDkwOC9wb2RjYXN0LnhtbA/episode/'
-        'MjA5MmZjM2ItYmMwZi00NGFiLWFlNDktM2I3YmFhMjA4ODVi?sa=X&ved=0CAUQkfYCahcKEwjg4s3umbjuAhUAAAAAHQAAAAAQAQ'
+    ft_google_ep_url = (
+        'https://podcasts.google.com/feed/aHR0cHM6Ly9yc3MuYWNhc3QuY29tL2Z0bmV3c2JyaWVmaW5n/episode/'
+        'NzM4Y2Q2NWEtMWM5Ni00Y2FjLWI5NDYtN2ExNGVmYThhOWRm?sa=X&ved=0CAUQkfYCahcKEwiI2eWUzcLxAhUAAAAAHQAAAAAQDg'
     )
 
-    assert _get_feed_url_from_google_podcasts_url(npr_google_ep_url) == npr_feed_url
+    assert _get_feed_url_from_google_podcasts_url(ft_google_ep_url) == ft_feed_url

apps/docker-compose.dist.yml

@@ -149,16 +149,6 @@ x-twitter-api-configuration: &twitter-api-configuration
     MC_TWITTER_ACCESS_TOKEN_SECRET: ""
 
 
-#
-# Crimson Hexagon API configuration
-# =================================
-#
-x-crimson-hexagon-api-configuration: &crimson-hexagon-api-configuration
-
-    # Crimson Hexagon API key
-    MC_CRIMSON_HEXAGON_API_KEY: ""
-
-
 #
 # Brandwatch API configuration
 # ============================
@@ -2073,7 +2063,6 @@ services:
         environment:
             <<: *common-configuration
             <<: *twitter-api-configuration
-            <<: *crimson-hexagon-api-configuration
             <<: *brandwatch-api-configuration
         deploy:
             <<: *misc-apps_deploy_placement_constraints
@@ -2099,7 +2088,6 @@ services:
         environment:
             <<: *common-configuration
             <<: *twitter-api-configuration
-            <<: *crimson-hexagon-api-configuration
             <<: *brandwatch-api-configuration
         deploy:
             <<: *misc-apps_deploy_placement_constraints

apps/topics-mine/docker-compose.tests.yml

@@ -7,7 +7,6 @@ services:
         init: true
         stop_signal: SIGKILL
         environment:
-            MC_CRIMSON_HEXAGON_API_KEY: "${MC_CRIMSON_HEXAGON_API_KEY}"
             MC_TWITTER_CONSUMER_KEY: "${MC_TWITTER_CONSUMER_KEY}"
             MC_TWITTER_CONSUMER_SECRET: "${MC_TWITTER_CONSUMER_SECRET}"
             MC_TWITTER_ACCESS_TOKEN: "${MC_TWITTER_ACCESS_TOKEN}"

apps/topics-mine/src/python/topics_mine/fetch_topic_posts.py

@@ -13,7 +13,6 @@
 from topics_mine.posts import AbstractPostFetcher
 from topics_mine.posts.archive_org_twitter import ArchiveOrgPostFetcher
 from topics_mine.posts.brandwatch_twitter import BrandwatchTwitterPostFetcher
-from topics_mine.posts.crimson_hexagon_twitter import CrimsonHexagonTwitterPostFetcher
 from topics_mine.posts.csv_generic import CSVStaticPostFetcher
 from topics_mine.posts.postgres_generic import PostgresPostFetcher
 from topics_mine.posts.pushshift_reddit import PushshiftRedditPostFetcher
@@ -219,8 +218,6 @@ def get_post_fetcher(topic_seed_query: dict) -> Optional[AbstractPostFetcher]:
 
     if source == 'brandwatch' and platform == 'twitter':
         fetch = BrandwatchTwitterPostFetcher()
-    elif source == 'crimson_hexagon' and platform == 'twitter':
-        fetch = CrimsonHexagonTwitterPostFetcher()
     elif source == 'postgres' and platform == 'generic_post':
         fetch = PostgresPostFetcher()
     elif source == 'csv' and platform == 'generic_post':

apps/topics-mine/src/python/topics_mine/posts/brandwatch_twitter.py

@@ -1,4 +1,4 @@
-"""Fetch twitter posts from crimson hexagon."""
+"""Fetch twitter posts from Brandwatch."""
 
 import datetime
 import dateutil
@@ -17,7 +17,6 @@
 from topics_base.posts import get_mock_data
 import topics_base.twitter as twitter
 from topics_base.twitter_url import get_tweet_urls 
-from topics_mine.config import TopicsMineConfig
 from topics_mine.posts import AbstractPostFetcher
 from topics_mine.posts.twitter.helpers import add_tweets_to_meta_tweets, get_tweet_id_from_url, McTwitterUrlException
 
@@ -41,7 +40,7 @@ def _mock_oauth(request, context) -> str:
     return '{"access_token":"foo","token_type":"bearer","expires_in":31535999,"scope":"read write trust"}'
 
 def _mock_posts(request, context) -> str:
-    """Mock crimson hexagon api call for requests_mock."""
+    """Mock Twitter api call for requests_mock."""
     params = parse_qs(urlparse(request.url).query)
 
     start_date = dateutil.parser.parse(params['startDate'][0])
@@ -182,7 +181,7 @@ def fetch_posts_from_api(
         sample: Optional[int] = None,
         page_size: Optional[int] = None,
     ) -> list:
-        """Fetch day of tweets from crimson hexagon and twitter."""
+        """Fetch day of tweets."""
 
         if page_size is None:
             page_size = 5000
@@ -241,7 +240,7 @@ def fetch_posts_from_api(
         return posts
 
     def setup_mock_data(self, mocker: requests_mock.Mocker) -> None:
-        """Fetch tweets from ch and twitter.  Setup mocking if self.mock_enabled."""
+        """Fetch tweets.  Setup mocking if self.mock_enabled."""
         # add the mockers for the bw api calls
         matcher = re.compile('.*api.brandwatch.com/oauth/token.*')
         mocker.post('https://api.brandwatch.com/oauth/token', text=_mock_oauth)