From 1382c0748da13eb203609a61c0a600d431c7633b Mon Sep 17 00:00:00 2001 From: John Ahlroos Date: Sun, 24 May 2026 18:36:05 +0200 Subject: [PATCH] Add svenska yle feed --- README.md | 3 +- app/main.py | 3 +- app/routers/yle_rss_sv.py | 141 ++++++++++++++++++++++++++++++++++++++ app/settings/defaults.py | 3 + compose.yml | 2 +- 5 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 app/routers/yle_rss_sv.py diff --git a/README.md b/README.md index 359e015..1dc426a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Automatically syncs RSS feeds from Finnish and international sources to your cus |--------|--------|----------|----------| | `yle_fi` | Yle.fi News | fi | Includes hashtags, custom cleaning | | `yle_en` | Yle.fi News | en | AI-generated translated tags | +| `yle_sv` | Svenska Yle | sv | Includes hashtags, custom cleaning | | `the_local` | The Local | en | Includes hashtags | | `taloustaito` | Taloustaito | fi | First 3 articles only | | `sur` | Sur Weather News | en | Includes hashtags | @@ -51,7 +52,7 @@ Automatically syncs RSS feeds from Finnish and international sources to your cus # - aws_access_key_id / aws_secret_access_key (optional) # Run development server with hot-reload -uvicorn app.main:app --host 0.0.0.0 --port 8000 +fastapi dev # Or use Docker Compose docker compose up --build diff --git a/app/main.py b/app/main.py index a9c4e6e..937c8a7 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,5 @@ from fastapi import Depends, FastAPI -from routers import embed, yle_rss_fi, yle_rss_en, the_local, taloustaito,sur,hackernews,fuengirola +from routers import embed, yle_rss_fi, yle_rss_en, yle_rss_sv, the_local, taloustaito,sur,hackernews,fuengirola from settings.defaults import get_settings @@ -9,6 +9,7 @@ app.include_router(embed.router, prefix="/embed", tags=["embed"]) app.include_router(yle_rss_fi.router, prefix="/rss", tags=["rss"]) app.include_router(yle_rss_en.router, prefix="/rss", tags=["rss"]) +app.include_router(yle_rss_sv.router, prefix="/rss", tags=["rss"]) app.include_router(the_local.router, prefix="/rss", tags=["rss"]) app.include_router(taloustaito.router, prefix="/rss", tags=["rss"]) app.include_router(sur.router, prefix="/rss", tags=["rss"]) diff --git a/app/routers/yle_rss_sv.py b/app/routers/yle_rss_sv.py new file mode 100644 index 0000000..eaef53a --- /dev/null +++ b/app/routers/yle_rss_sv.py @@ -0,0 +1,141 @@ +import traceback +import json +import requests +import traceback +import json +import re +import feedparser +import requests +import logging + +from datetime import datetime +from time import mktime +from typing import Annotated +from fastapi import Depends, APIRouter + +from settings.defaults import Settings, get_settings + +router = APIRouter() +logger = logging.getLogger(__name__) + +@router.get("/yle_sv", summary="Svenska Yle RSS") +async def update(settings: Annotated[Settings, Depends(get_settings)]): + + feed_url = settings.feeds['yle_sv']['url'] + + mastodon_server = settings.mastodon_server + mastodon_aid = settings.feeds['yle_sv']['account_id'] + mastodon_token = settings.feeds['yle_sv']['token'] + mastodon_get_statuses_url=f'{mastodon_server}/api/v1/accounts/{mastodon_aid}/statuses' + mastodon_post_statuses_url=f'{mastodon_server}/api/v1/statuses' + + try: + last_status_timestamp=datetime.fromisoformat(load_last_status(mastodon_get_statuses_url, mastodon_token)['created_at']) + new_entries=load_feed_rss(feed_url, last_status_timestamp) + logger.info(f'Found {len(new_entries)} new entries since {last_status_timestamp}') + + if (len(new_entries) == 0): + return { + "status": 200, + "body": { + "posted_entries": 0, + "successful": True + } + } + + posted_entries=list(map(lambda x: post_rss_entry_to_mastodon(mastodon_post_statuses_url, mastodon_token, x), new_entries)) + + return { + "status": 200, + "body": { + "posted_entries": len(posted_entries), + "successful": True + } + } + except Exception as e: + msg = ''.join(traceback.format_exception_only(e)) + logger.error(msg) + return { + "status": 501, + "body": { + "posted_entries": 0, + "message": msg, + "successful": False + } + } + +def split(arr, char): + return [tag for subtags in (map(lambda str: str.split(char), arr)) for tag in subtags] + +def capitalize(arr, char): + result = map(lambda str: str.split(char), arr) + result = map(lambda subtag: map(lambda str: str.capitalize(), subtag), result) + result = map(lambda subtag: ''.join(subtag), result) + return result + +def load_last_status(url, token): + response=requests.get(url + '?limit=1', headers={ 'Authorization' : f'Bearer {token}' }) + if response.status_code != 200: + raise Exception('Failed to contact Mastodon', response.text) + + body = json.loads(response.text) + if len(body) == 0: + return json.loads('{ "created_at": "2000-01-01 00:00:00"}') + + return body[0] + + +def post_rss_entry_to_mastodon(url, token, entry): + + title = entry.title + description = entry.summary + link = entry.link + + linkEnd = entry.link.find('?') + if linkEnd > -1: + link = entry.link[0:linkEnd] + else: + link = entry.link + + if 'tags' in entry: + categories = [t.get('term') for t in entry.tags] + categories = split(categories, ',') + categories = capitalize(categories, ' ') + categories = capitalize(categories, '–') + categories = capitalize(categories, '-') + categories = capitalize(categories, '/') + categories = capitalize(categories, '\\') + categories = map(lambda str: re.sub(r'\s+','', str), categories) + categories = map(lambda str: re.sub(r'[0-9.()]+','', str), categories) + categories = map(lambda str: re.sub('&','och', str), categories) + + categories = [str for str in categories if len(str) >= 3] + if len(categories) > 0: + categories = map(lambda str: str if str.startswith('#') else f'#{str}', categories) + categories = ' '.join(categories) + message = f"{title}\n\n{description}\n\n{link}\n\n{categories}" + else: + message = f"{title}\n\n{description}\n\n{link}" + else: + message = f"{title}\n\n{description}\n\n{link}" + + headers = { + 'Authorization': f'Bearer {token}', + 'Content-type': 'application/x-www-form-urlencoded', + 'User-Agent': 'Serverless Feed' + } + + params = { + 'status': message, + 'language': 'sv', + 'visibility': 'public' + } + + response = requests.post(url, data=params, headers=headers) + if response.status_code != 200: + logger.error('Failed to post message', response) + return response + +def load_feed_rss(url, since): + feed=feedparser.parse(url) + return [entry for entry in feed.entries if datetime.fromtimestamp(mktime(entry.published_parsed)) > since.replace(tzinfo=datetime.fromtimestamp(mktime(entry.published_parsed)).tzinfo)] diff --git a/app/settings/defaults.py b/app/settings/defaults.py index 6115767..186922d 100644 --- a/app/settings/defaults.py +++ b/app/settings/defaults.py @@ -4,6 +4,9 @@ from functools import lru_cache class Settings(BaseSettings): mastodon_server:str openai_api_key:str + aws_access_key_id: str + aws_secret_access_key: str + aws_endpoint_url_s3: str feeds: dict[str, dict[str,object]] model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter='__', arbitrary_types_allowed=True) version:str diff --git a/compose.yml b/compose.yml index 7658172..9affd09 100644 --- a/compose.yml +++ b/compose.yml @@ -12,7 +12,7 @@ services: - 8000:8000 develop: watch: - - action: sync + - action: rebuild path: ./app target: /code/app - action: rebuild