python test_soundcloud_embeds.py

Posted 2021-05-09

from __future__ import absolute_import

from datetime import datetime, timedelta
import re
from django.core.management.base import BaseCommand

from apple_news.exceptions import AppleNewsIncompatibleFormatError
from apple_news.process import AppleNewsProcessor

from blog.models import BlogArticle
from magazine.models import Article

from lxml.html import fragment_fromstring, etree
from ollie.utils.html import cleanup_html
from apple_news.process import remove_soundcloud_embeds


"""
This command tests whether or not errors being thrown from AppleNews publishing
are being caused from iFrame unexpected sources.  For cases where they are soundcloud 
sources (i.e. Audm embeds), it will try to remove the soundcloud embed and retest the publish
if --fix=True.  Note that in order to test the functionality when soundcloud embeds throw errors,
you must remove the call remove_soundcloud_embeds in apple_news/process.py
"""
class Command(BaseCommand):

    help = """Processes the last 60 days of Apple News material and provides
            list of errors that occur when apple_news.AppleNewsHtmlConverter
            does not correctly convert the news item article"""

    def add_arguments(self, parser):
        parser.add_argument(
            '--days',
            type=int,
            help='Number of days to lookback')
        parser.add_argument(
            '--fix',
            type=bool,
            help='Whether or not to try to remove soundcloud embeds before processing.')

    def handle(self, days=30, fix=False, **options):

        end = datetime.today()
        start = end - timedelta(days=days)
        end = end.strftime("%Y-%m-%d")
        start = start.strftime("%Y-%m-%d")

        self.test_blog_articles(start, end, fix=fix)
        self.test_magazine_articles(start, end, fix=fix)

    def test_blog_articles(self, start, end, fix=False):
        Message.print_color('Testing Blog Articles Between {} and {}'.format(start, end))
        test = ArticleTest(fix=fix, name='Blog')

        blogs = BlogArticle.objects.filter(status=3).filter(
            date_published__range=[start, end]
        ).all()

        for blog in blogs:
            test(blog)

    def test_magazine_articles(self, start, end, fix=False):
        Message.print_color('Testing Magazine Articles Between {} and {}'.format(start, end))
        test = ArticleTest(fix=fix, name='Magazine')

        magazines = Article.objects.filter(status=3).filter(
            date_published__range=[start, end]
        ).all()

        for magazine in magazines:
            test(magazine)


class Message:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    OK = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

    @staticmethod
    def sub_message(text, style=None):
        try:
            message = '   ---> {}'.format(text.title())
        except AttributeError:
            message = '   ---> {}'.format(text)
        Message.print_color(message, style=style)

    @staticmethod
    def sub_sub_message(text, style=None):
        try:
            message = '      ---> {}'.format(text.title())
        except AttributeError:
            message = '      ---> {}'.format(text)
        Message.print_color(message, style=style)

    @staticmethod
    def color(text, style=None):
        if not style:
            return text
        prefix = getattr(Message, style.upper())
        return prefix + text + Message.ENDC

    @staticmethod
    def print_color(text, style=None, newline=False):
        message = Message.color(text, style=style)
        if newline:
            message = '\n' + message 
        print message


class ArticleTest(object):
    def __init__(self, fix=False, name=''):
        self.fix = fix
        self.name = name

    def test_fix(self, article):
        html = re.sub(r"\s+", ' ', article.content)
        root = fragment_fromstring(html, create_parent=True)
        cleanup_html(root)

        remove_soundcloud_embeds(root)
        cleanup_html(root)
        article.content = etree.tostring(root)
        return article

    def test_publish(self, article):
        native = AppleNewsProcessor(article)
        try:
            native.to_bundle()
            return None 
        except AppleNewsIncompatibleFormatError as e:
            return e 
        return

    def __call__(self, model):
        error = self.test_publish(model)
        if error:
            title = model.title.encode('ascii', 'ignore')
            header = '{} Article {} - {}'.format(self.name.title(), title, str(model.pk))
            Message.print_color(header, style='underline', newline=True)

            if "iFrame with unexpected" in str(error):
                Message.sub_message('Found error with iFrame', style='warning')

                if 'soundcloud' in str(error).lower():
                    Message.sub_message('Error is from Soundcloud Audm Embed...', style='bold')
                    if self.fix:
                        model = self.test_fix(model)
                        error = self.test_publish(model)
                        if error:
                            Message.sub_sub_message('Could not fix the error for {}'.format(model.pk), style='fail')
                            Message.sub_sub_message(str(error))
                        else:
                            Message.sub_sub_message('Fixed the error for {}'.format(model.pk), style='ok')
                else:
                    Message.sub_message('Error is not from Soundcloud Audm Embed but from other iFrame Media..', style='bold')
            else:
                Message.sub_message('Found another error', style='fail')
                Message.sub_sub_message(str(error), style='bold')