From b1bbc1e0243a679fb1be2361d5bde3bd5f98077b Mon Sep 17 00:00:00 2001 From: kevincaires Date: Wed, 7 Sep 2022 13:47:13 -0300 Subject: [PATCH 1/9] ADD create files_bloob table and try to add fk column in neeble_quotes. --- utils/database.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/utils/database.py b/utils/database.py index ec34037..83ca981 100644 --- a/utils/database.py +++ b/utils/database.py @@ -49,13 +49,30 @@ def migrate() -> None: user varchar(200) not null, quote varchar(500) not null unique, index quote_idx (quote) - ); + + ) character set utf8mb4 collate utf8mb4_general_ci; + + create table if not exists files_bloob( + id int auto_increment primary key, + file blob not null + ) character set utf8mb4 collate utf8mb4_general_ci; ''' try: with Cursor(MYSQL_CONFIG) as cursor: cursor.execute(_sql) except Exception as ex: logger.error(ex.args) + else: + _sql = ''' + alter table neeble_quotes + add column file_id int unique null, + add foreign key file_id(neeble_quotes) references files_bloob(id) on delete cascade; + ''' + try: + with Cursor(MYSQL_CONFIG) as cursor: + cursor.execute(_sql) + except Exception as ex: + logger.error(ex.args) def set_quote(user: str, quote: str) -> int: From e4421a12319cc3afd8ad0d2ad7d3e6d14f08f3b7 Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Thu, 22 Sep 2022 10:25:52 -0300 Subject: [PATCH 2/9] ADD viewed news table to control the viewed news. --- models/news.py | 16 ++++++++++++++++ utils/database.py | 18 ++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 models/news.py diff --git a/models/news.py b/models/news.py new file mode 100644 index 0000000..5270681 --- /dev/null +++ b/models/news.py @@ -0,0 +1,16 @@ +from settings.config import SQLACHEMY +from sqlalchemy import Table +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +class ViewedNews(Base): + """ + Viewed news model class. + """ + __table__ = Table( + "viewed_news", + Base.metadata, + autoload=True, + autoload_with=SQLACHEMY + ) diff --git a/utils/database.py b/utils/database.py index 83ca981..8e7ebc9 100644 --- a/utils/database.py +++ b/utils/database.py @@ -52,9 +52,11 @@ def migrate() -> None: ) character set utf8mb4 collate utf8mb4_general_ci; - create table if not exists files_bloob( + create table if not exists viewed_news( id int auto_increment primary key, - file blob not null + title varchar(500) not null unique, + published_at date not null, + index viewed_idx(title, published_at) ) character set utf8mb4 collate utf8mb4_general_ci; ''' try: @@ -62,24 +64,12 @@ def migrate() -> None: cursor.execute(_sql) except Exception as ex: logger.error(ex.args) - else: - _sql = ''' - alter table neeble_quotes - add column file_id int unique null, - add foreign key file_id(neeble_quotes) references files_bloob(id) on delete cascade; - ''' - try: - with Cursor(MYSQL_CONFIG) as cursor: - cursor.execute(_sql) - except Exception as ex: - logger.error(ex.args) def set_quote(user: str, quote: str) -> int: """ Set a quote into database. """ - qt = Quotes(quote=quote, user=user,) qtid = 0 with Session(SQLACHEMY) as session: From ab5313081dbc27e493ef5581de82e24fe043f484 Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Thu, 22 Sep 2022 10:26:04 -0300 Subject: [PATCH 3/9] ADD request. --- requirements/common.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/common.txt b/requirements/common.txt index 189270c..be3662d 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -3,3 +3,4 @@ mysqlclient psutil SQLAlchemy pytz +requests From b70fbcaa70fcd00251019c9ca70cfd34fe759dab Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Thu, 22 Sep 2022 10:27:08 -0300 Subject: [PATCH 4/9] CHANGE to get everything and add some util methods. --- utils/news_paper.py | 55 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/utils/news_paper.py b/utils/news_paper.py index a0c0297..f4badaf 100644 --- a/utils/news_paper.py +++ b/utils/news_paper.py @@ -1,6 +1,10 @@ import logging +from datetime import datetime + import requests -from settings.config import GOOGLE_NEWS +from models.news import ViewedNews +from settings.config import GOOGLE_NEWS, SQLACHEMY +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -9,7 +13,7 @@ class News: """ Get the information in IBGE API. """ - _url = f'{GOOGLE_NEWS["url"]}top-headlines?'\ + _url = f'{GOOGLE_NEWS["url"]}everything?'\ f'sources={",".join(GOOGLE_NEWS["sources"])}'\ f'&apiKey={GOOGLE_NEWS["token"]}' @@ -19,6 +23,46 @@ class News: """ self.quantity = int(quantity) + def _date_convert(self, _date) -> datetime: + """ + Convert new datetime. + """ + try: + _date = datetime.fromisoformat(_date) + except ValueError: + _date = datetime.strptime(_date, '%Y-%m-%dT%H:%M:%Sz') + except Exception as e: + logger.error(e) + + return _date + + def _remove_viewed_news(self, articles: list) -> list: + """ + Remove the viewed news. + """ + with Session(SQLACHEMY) as session: + for idx, new in enumerate(articles): + viewed = session.query(ViewedNews).filter( + ViewedNews.title==new['title'], + ViewedNews.published_at==self._date_convert(new['publishedAt']) + ) + + if list(viewed): + articles.pop(idx) + + return articles + + def _set_viewed_news(self, articles: list) -> None: + """ + Set viewed news in data base. + """ + with Session(SQLACHEMY) as session: + for article in articles: + _date = self._date_convert(article['publishedAt']) + new = ViewedNews(title=article['title'], published_at=_date.date()) + session.add(new) + session.commit() + def news(self) -> list: """ Get the information based in self.quantity attribute. @@ -30,7 +74,12 @@ class News: logger.error(content) raise Exception(content) - return content['articles'][:self.quantity] + # Remove viewed news. + response = self._remove_viewed_news(content['articles'])[:self.quantity] + # Save news into a blacklist. + self._set_viewed_news(response) + + return response def filter(self, phrase: str) -> list: """ From 950f1fb0154895805a0631ad7f6d204f530f6c8d Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Wed, 28 Sep 2022 08:50:52 -0300 Subject: [PATCH 5/9] FIX remove infomoney from google news. --- settings/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/config.py b/settings/config.py index 62bf23d..5ea20ff 100644 --- a/settings/config.py +++ b/settings/config.py @@ -100,5 +100,5 @@ IMAGE_TYPES = ( GOOGLE_NEWS = { 'url': os.environ.get('GOOGLE_NEWS_API'), 'token': os.environ.get('GOOGLE_NEWS_TOKEN'), - 'sources': ['google-news-br', 'info-money', 'globo'] + 'sources': ['google-news-br', 'globo'] } From 3e1e8144deb485fd8afbe0d940329ab070e990a8 Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Wed, 28 Sep 2022 09:22:36 -0300 Subject: [PATCH 6/9] FIX _remove_viewed_news function; --- utils/news_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/news_paper.py b/utils/news_paper.py index f4badaf..14a8dea 100644 --- a/utils/news_paper.py +++ b/utils/news_paper.py @@ -44,7 +44,7 @@ class News: for idx, new in enumerate(articles): viewed = session.query(ViewedNews).filter( ViewedNews.title==new['title'], - ViewedNews.published_at==self._date_convert(new['publishedAt']) + ViewedNews.published_at==self._date_convert(new['publishedAt']).date() ) if list(viewed): From 4b215eff18a368afb0a957146527d2efb87ad364 Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Wed, 28 Sep 2022 09:29:21 -0300 Subject: [PATCH 7/9] ADD Integrity error exception; --- utils/news_paper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/news_paper.py b/utils/news_paper.py index 14a8dea..ccabe6e 100644 --- a/utils/news_paper.py +++ b/utils/news_paper.py @@ -5,6 +5,7 @@ import requests from models.news import ViewedNews from settings.config import GOOGLE_NEWS, SQLACHEMY from sqlalchemy.orm import Session +from sqlalchemy.exc import IntegrityError logger = logging.getLogger(__name__) @@ -61,7 +62,11 @@ class News: _date = self._date_convert(article['publishedAt']) new = ViewedNews(title=article['title'], published_at=_date.date()) session.add(new) - session.commit() + + try: + session.commit() + except IntegrityError: + continue def news(self) -> list: """ From b6ec9010d5d87ffaf918617d8cdf31f3abd153b5 Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Wed, 28 Sep 2022 09:57:54 -0300 Subject: [PATCH 8/9] ADD except for unknown errors. --- utils/news_paper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/news_paper.py b/utils/news_paper.py index ccabe6e..8d1bece 100644 --- a/utils/news_paper.py +++ b/utils/news_paper.py @@ -66,7 +66,11 @@ class News: try: session.commit() except IntegrityError: + logger.warning('Duplicated entry.\nTitle: %s' % (article['title'])) continue + except Exception as e: + logger.error(e.args[0]) + pass def news(self) -> list: """ From f83618f5cddd21fc5a3ffa58f50b906e6b128dcf Mon Sep 17 00:00:00 2001 From: "kevin.caires" Date: Wed, 28 Sep 2022 09:58:30 -0300 Subject: [PATCH 9/9] ADD datetime treatment. --- utils/commands.py | 17 +++-------------- utils/tools.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/utils/commands.py b/utils/commands.py index 3261b8a..813a72e 100644 --- a/utils/commands.py +++ b/utils/commands.py @@ -2,10 +2,8 @@ Bot commands. """ import logging -from datetime import datetime from random import choice -import pytz from discord import Embed, Intents from discord.ext import commands from settings.config import IMAGE_TYPES, OW_API_CONFIG, PERMISSIONS @@ -14,7 +12,7 @@ from utils.database import (count_quotes, get_by_id, get_quote_contains, get_quotes, remove_quote, set_quote) from utils.machine_monitor import Monitor from utils.news_paper import News -from utils.tools import kbytes_to_gbytes +from utils.tools import datetime_to_string, kbytes_to_gbytes from utils.weather import displayweather, getweatherdata client = commands.Bot(command_prefix='--', intents=Intents.all()) @@ -345,18 +343,9 @@ async def news(bot: object, *options: str) -> None: return for new in news: - try: - dt = datetime.fromisoformat( - new['publishedAt'] - ).astimezone(pytz.timezone('America/Sao_Paulo')) - except ValueError: - dt = datetime.strptime(new['publishedAt'], '%Y-%m-%dT%H:%M:%Sz') - except Exception as e: - logger.error(e) - continue - + dt = datetime_to_string(new['publishedAt']) embed.add_field(name='Font', value=new['source']['name'], inline=False) - embed.add_field(name='Published at', value=dt.isoformat(), inline=False) + embed.add_field(name='Published at', value=dt, inline=False) embed.add_field(name='Link', value=new['url'], inline=False) embed.add_field(name=new['title'], value=new['description'], inline=False) embed.add_field(name='---', value='---') diff --git a/utils/tools.py b/utils/tools.py index 1400273..69ff48a 100644 --- a/utils/tools.py +++ b/utils/tools.py @@ -1,9 +1,42 @@ """ Tools module. """ +import logging +from datetime import datetime + +import pytz + +logger = logging.getLogger(__name__) + + def kbytes_to_gbytes(value: float) -> str: """ Transform Kb into Gb. """ _value = value / (1024 ** 3) return "{:.2f}".format(_value) + + +def datetime_to_string(_datetime: datetime) -> str: + """ + Transform datetime in string DD/MM/AAAA HH:MM:SS + """ + try: + dt = datetime.fromisoformat( + _datetime + ) + except ValueError: + dt = datetime.strptime( + _datetime, + '%Y-%m-%dT%H:%M:%Sz' + ) + except Exception as e: + logger.error(e) + return dt + + try: + dt = dt.astimezone(pytz.timezone('America/Sao_Paulo')) + except Exception as e: + logger.error(e) + else: + return dt.date().isoformat()