diff --git a/models/news.py b/models/news.py new file mode 100644 index 0000000..5270681 --- /dev/null +++ b/models/news.py @@ -0,0 +1,16 @@ +from settings.config import SQLACHEMY +from sqlalchemy import Table +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +class ViewedNews(Base): + """ + Viewed news model class. + """ + __table__ = Table( + "viewed_news", + Base.metadata, + autoload=True, + autoload_with=SQLACHEMY + ) diff --git a/requirements/common.txt b/requirements/common.txt index 189270c..be3662d 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -3,3 +3,4 @@ mysqlclient psutil SQLAlchemy pytz +requests diff --git a/settings/config.py b/settings/config.py index 62bf23d..5ea20ff 100644 --- a/settings/config.py +++ b/settings/config.py @@ -100,5 +100,5 @@ IMAGE_TYPES = ( GOOGLE_NEWS = { 'url': os.environ.get('GOOGLE_NEWS_API'), 'token': os.environ.get('GOOGLE_NEWS_TOKEN'), - 'sources': ['google-news-br', 'info-money', 'globo'] + 'sources': ['google-news-br', 'globo'] } diff --git a/utils/commands.py b/utils/commands.py index 3261b8a..813a72e 100644 --- a/utils/commands.py +++ b/utils/commands.py @@ -2,10 +2,8 @@ Bot commands. """ import logging -from datetime import datetime from random import choice -import pytz from discord import Embed, Intents from discord.ext import commands from settings.config import IMAGE_TYPES, OW_API_CONFIG, PERMISSIONS @@ -14,7 +12,7 @@ from utils.database import (count_quotes, get_by_id, get_quote_contains, get_quotes, remove_quote, set_quote) from utils.machine_monitor import Monitor from utils.news_paper import News -from utils.tools import kbytes_to_gbytes +from utils.tools import datetime_to_string, kbytes_to_gbytes from utils.weather import displayweather, getweatherdata client = commands.Bot(command_prefix='--', intents=Intents.all()) @@ -345,18 +343,9 @@ async def news(bot: object, *options: str) -> None: return for new in news: - try: - dt = datetime.fromisoformat( - new['publishedAt'] - ).astimezone(pytz.timezone('America/Sao_Paulo')) - except ValueError: - dt = datetime.strptime(new['publishedAt'], '%Y-%m-%dT%H:%M:%Sz') - except Exception as e: - logger.error(e) - continue - + dt = datetime_to_string(new['publishedAt']) embed.add_field(name='Font', value=new['source']['name'], inline=False) - embed.add_field(name='Published at', value=dt.isoformat(), inline=False) + embed.add_field(name='Published at', value=dt, inline=False) embed.add_field(name='Link', value=new['url'], inline=False) embed.add_field(name=new['title'], value=new['description'], inline=False) embed.add_field(name='---', value='---') diff --git a/utils/database.py b/utils/database.py index ec34037..8e7ebc9 100644 --- a/utils/database.py +++ b/utils/database.py @@ -49,7 +49,15 @@ def migrate() -> None: user varchar(200) not null, quote varchar(500) not null unique, index quote_idx (quote) - ); + + ) character set utf8mb4 collate utf8mb4_general_ci; + + create table if not exists viewed_news( + id int auto_increment primary key, + title varchar(500) not null unique, + published_at date not null, + index viewed_idx(title, published_at) + ) character set utf8mb4 collate utf8mb4_general_ci; ''' try: with Cursor(MYSQL_CONFIG) as cursor: @@ -62,7 +70,6 @@ def set_quote(user: str, quote: str) -> int: """ Set a quote into database. """ - qt = Quotes(quote=quote, user=user,) qtid = 0 with Session(SQLACHEMY) as session: diff --git a/utils/news_paper.py b/utils/news_paper.py index a0c0297..8d1bece 100644 --- a/utils/news_paper.py +++ b/utils/news_paper.py @@ -1,6 +1,11 @@ import logging +from datetime import datetime + import requests -from settings.config import GOOGLE_NEWS +from models.news import ViewedNews +from settings.config import GOOGLE_NEWS, SQLACHEMY +from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError logger = logging.getLogger(__name__) @@ -9,7 +14,7 @@ class News: """ Get the information in IBGE API. """ - _url = f'{GOOGLE_NEWS["url"]}top-headlines?'\ + _url = f'{GOOGLE_NEWS["url"]}everything?'\ f'sources={",".join(GOOGLE_NEWS["sources"])}'\ f'&apiKey={GOOGLE_NEWS["token"]}' @@ -19,6 +24,54 @@ class News: """ self.quantity = int(quantity) + def _date_convert(self, _date) -> datetime: + """ + Convert new datetime. + """ + try: + _date = datetime.fromisoformat(_date) + except ValueError: + _date = datetime.strptime(_date, '%Y-%m-%dT%H:%M:%Sz') + except Exception as e: + logger.error(e) + + return _date + + def _remove_viewed_news(self, articles: list) -> list: + """ + Remove the viewed news. + """ + with Session(SQLACHEMY) as session: + for idx, new in enumerate(articles): + viewed = session.query(ViewedNews).filter( + ViewedNews.title==new['title'], + ViewedNews.published_at==self._date_convert(new['publishedAt']).date() + ) + + if list(viewed): + articles.pop(idx) + + return articles + + def _set_viewed_news(self, articles: list) -> None: + """ + Set viewed news in data base. + """ + with Session(SQLACHEMY) as session: + for article in articles: + _date = self._date_convert(article['publishedAt']) + new = ViewedNews(title=article['title'], published_at=_date.date()) + session.add(new) + + try: + session.commit() + except IntegrityError: + logger.warning('Duplicated entry.\nTitle: %s' % (article['title'])) + continue + except Exception as e: + logger.error(e.args[0]) + pass + def news(self) -> list: """ Get the information based in self.quantity attribute. @@ -30,7 +83,12 @@ class News: logger.error(content) raise Exception(content) - return content['articles'][:self.quantity] + # Remove viewed news. + response = self._remove_viewed_news(content['articles'])[:self.quantity] + # Save news into a blacklist. + self._set_viewed_news(response) + + return response def filter(self, phrase: str) -> list: """ diff --git a/utils/tools.py b/utils/tools.py index 1400273..69ff48a 100644 --- a/utils/tools.py +++ b/utils/tools.py @@ -1,9 +1,42 @@ """ Tools module. """ +import logging +from datetime import datetime + +import pytz + +logger = logging.getLogger(__name__) + + def kbytes_to_gbytes(value: float) -> str: """ Transform Kb into Gb. """ _value = value / (1024 ** 3) return "{:.2f}".format(_value) + + +def datetime_to_string(_datetime: datetime) -> str: + """ + Transform datetime in string DD/MM/AAAA HH:MM:SS + """ + try: + dt = datetime.fromisoformat( + _datetime + ) + except ValueError: + dt = datetime.strptime( + _datetime, + '%Y-%m-%dT%H:%M:%Sz' + ) + except Exception as e: + logger.error(e) + return dt + + try: + dt = dt.astimezone(pytz.timezone('America/Sao_Paulo')) + except Exception as e: + logger.error(e) + else: + return dt.date().isoformat()