Merge pull request 'vinke' (#20) from vinke into develop

Reviewed-on: http://vps41059.publiccloud.com.br:3000/neeble-club/neeble/pulls/20

models/news.py

@@ -0,0 +1,16 @@
+from settings.config import SQLACHEMY
+from sqlalchemy import Table
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+class ViewedNews(Base):
+    """
+    Viewed news model class.
+    """
+    __table__ = Table(
+        "viewed_news",
+        Base.metadata,
+        autoload=True,
+        autoload_with=SQLACHEMY
+    )

requirements/common.txt

@@ -3,3 +3,4 @@ mysqlclient
 psutil
 SQLAlchemy
 pytz
+requests

settings/config.py

@@ -100,5 +100,5 @@ IMAGE_TYPES = (
 GOOGLE_NEWS = {
     'url': os.environ.get('GOOGLE_NEWS_API'),
     'token': os.environ.get('GOOGLE_NEWS_TOKEN'),
-    'sources': ['google-news-br', 'info-money', 'globo']
+    'sources': ['google-news-br', 'globo']
 }

utils/commands.py

@@ -2,10 +2,8 @@
 Bot commands.
 """
 import logging
-from datetime import datetime
 from random import choice
 
-import pytz
 from discord import Embed, Intents
 from discord.ext import commands
 from settings.config import IMAGE_TYPES, OW_API_CONFIG, PERMISSIONS
@@ -14,7 +12,7 @@ from utils.database import (count_quotes, get_by_id, get_quote_contains,
                             get_quotes, remove_quote, set_quote)
 from utils.machine_monitor import Monitor
 from utils.news_paper import News
-from utils.tools import kbytes_to_gbytes
+from utils.tools import datetime_to_string, kbytes_to_gbytes
 from utils.weather import displayweather, getweatherdata
 
 client = commands.Bot(command_prefix='--', intents=Intents.all())
@@ -345,18 +343,9 @@ async def news(bot: object, *options: str) -> None:
         return
 
     for new in news:
-        try:
+        dt = datetime_to_string(new['publishedAt'])
-            dt = datetime.fromisoformat(
-                new['publishedAt']
-            ).astimezone(pytz.timezone('America/Sao_Paulo'))
-        except ValueError:
-            dt = datetime.strptime(new['publishedAt'], '%Y-%m-%dT%H:%M:%Sz')
-        except Exception as e:
-            logger.error(e) 
-            continue
-
         embed.add_field(name='Font', value=new['source']['name'], inline=False)
-        embed.add_field(name='Published at', value=dt.isoformat(), inline=False)
+        embed.add_field(name='Published at', value=dt, inline=False)
         embed.add_field(name='Link', value=new['url'], inline=False)
         embed.add_field(name=new['title'], value=new['description'], inline=False)
         embed.add_field(name='---', value='---')

utils/database.py

@@ -49,7 +49,15 @@ def migrate() -> None:
             user varchar(200) not null,
             quote varchar(500) not null unique,
             index quote_idx (quote)
-        );
+            
+        ) character set utf8mb4 collate utf8mb4_general_ci;
+
+        create table if not exists viewed_news(
+            id int auto_increment primary key,
+            title varchar(500) not null unique,
+            published_at date not null,
+            index viewed_idx(title, published_at)
+        ) character set utf8mb4 collate utf8mb4_general_ci;
     '''
     try:
         with Cursor(MYSQL_CONFIG) as cursor:
@@ -62,7 +70,6 @@ def set_quote(user: str, quote: str) -> int:
     """
     Set a quote into database.
     """
-
     qt = Quotes(quote=quote, user=user,)
     qtid = 0
     with Session(SQLACHEMY) as session:

utils/news_paper.py

@@ -1,6 +1,11 @@
 import logging
+from datetime import datetime
+
 import requests
-from settings.config import GOOGLE_NEWS
+from models.news import ViewedNews
+from settings.config import GOOGLE_NEWS, SQLACHEMY
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import IntegrityError
 
 logger = logging.getLogger(__name__)
 
@@ -9,7 +14,7 @@ class News:
     """
     Get the information in IBGE API.
     """
-    _url = f'{GOOGLE_NEWS["url"]}top-headlines?'\
+    _url = f'{GOOGLE_NEWS["url"]}everything?'\
         f'sources={",".join(GOOGLE_NEWS["sources"])}'\
         f'&apiKey={GOOGLE_NEWS["token"]}'
 
@@ -19,6 +24,54 @@ class News:
         """
         self.quantity = int(quantity)
 
+    def _date_convert(self, _date) -> datetime:
+        """
+        Convert new datetime.
+        """
+        try:
+            _date = datetime.fromisoformat(_date)
+        except ValueError:
+            _date = datetime.strptime(_date, '%Y-%m-%dT%H:%M:%Sz')
+        except Exception as e:
+            logger.error(e)
+
+        return _date
+
+    def _remove_viewed_news(self, articles: list) -> list:
+        """
+        Remove the viewed news. 
+        """
+        with Session(SQLACHEMY) as session:
+            for idx, new in enumerate(articles):
+                viewed = session.query(ViewedNews).filter(
+                    ViewedNews.title==new['title'],
+                    ViewedNews.published_at==self._date_convert(new['publishedAt']).date()
+                )
+
+                if list(viewed):
+                    articles.pop(idx)
+
+        return articles
+
+    def _set_viewed_news(self, articles: list) -> None:
+        """
+        Set viewed news in data base.
+        """
+        with Session(SQLACHEMY) as session:
+            for article in articles:
+                _date = self._date_convert(article['publishedAt'])
+                new = ViewedNews(title=article['title'], published_at=_date.date())
+                session.add(new)
+
+                try:
+                    session.commit()
+                except IntegrityError:
+                    logger.warning('Duplicated entry.\nTitle: %s' % (article['title']))
+                    continue
+                except Exception as e:
+                    logger.error(e.args[0])
+                    pass
+
     def news(self) -> list:
         """
         Get the information based in self.quantity attribute.
@@ -30,7 +83,12 @@ class News:
             logger.error(content)
             raise Exception(content)
 
-        return content['articles'][:self.quantity]
+        # Remove viewed news.
+        response = self._remove_viewed_news(content['articles'])[:self.quantity]
+        # Save news into a blacklist.
+        self._set_viewed_news(response)
+
+        return response
 
     def filter(self, phrase: str) -> list:
         """

utils/tools.py

@@ -1,9 +1,42 @@
 """
 Tools module.
 """
+import logging
+from datetime import datetime
+
+import pytz
+
+logger = logging.getLogger(__name__)
+
+
 def kbytes_to_gbytes(value: float) -> str:
     """
     Transform Kb into Gb.
     """
     _value = value / (1024 ** 3)
     return "{:.2f}".format(_value)
+
+
+def datetime_to_string(_datetime: datetime) -> str:
+    """
+    Transform datetime in string DD/MM/AAAA HH:MM:SS
+    """
+    try:
+        dt = datetime.fromisoformat(
+            _datetime
+        )
+    except ValueError:
+        dt = datetime.strptime(
+            _datetime,
+            '%Y-%m-%dT%H:%M:%Sz'
+        )
+    except Exception as e:
+        logger.error(e) 
+        return dt
+
+    try:
+        dt = dt.astimezone(pytz.timezone('America/Sao_Paulo'))
+    except Exception as e:
+        logger.error(e)
+    else:
+        return dt.date().isoformat()