Add svenska yle feed

README.md

@@ -16,6 +16,7 @@ Automatically syncs RSS feeds from Finnish and international sources to your cus
 |--------|--------|----------|----------|
 | `yle_fi` | Yle.fi News | fi | Includes hashtags, custom cleaning |
 | `yle_en` | Yle.fi News | en | AI-generated translated tags |
+| `yle_sv` | Svenska Yle | sv | Includes hashtags, custom cleaning |
 | `the_local` | The Local | en | Includes hashtags |
 | `taloustaito` | Taloustaito | fi | First 3 articles only |
 | `sur` | Sur Weather News | en | Includes hashtags |
@@ -51,7 +52,7 @@ Automatically syncs RSS feeds from Finnish and international sources to your cus
 # - aws_access_key_id / aws_secret_access_key (optional)
 
 # Run development server with hot-reload
-uvicorn app.main:app --host 0.0.0.0 --port 8000
+fastapi dev
 
 # Or use Docker Compose
 docker compose up --build

app/main.py

@@ -1,5 +1,5 @@
 from fastapi import Depends, FastAPI
-from routers import embed, yle_rss_fi, yle_rss_en, the_local, taloustaito,sur,hackernews,fuengirola
+from routers import embed, yle_rss_fi, yle_rss_en, yle_rss_sv, the_local, taloustaito,sur,hackernews,fuengirola
 
 from settings.defaults import get_settings
 
@@ -9,6 +9,7 @@ app.include_router(embed.router, prefix="/embed", tags=["embed"])
 
 app.include_router(yle_rss_fi.router, prefix="/rss", tags=["rss"])
 app.include_router(yle_rss_en.router, prefix="/rss", tags=["rss"])
+app.include_router(yle_rss_sv.router, prefix="/rss", tags=["rss"])
 app.include_router(the_local.router, prefix="/rss", tags=["rss"])
 app.include_router(taloustaito.router, prefix="/rss", tags=["rss"])
 app.include_router(sur.router, prefix="/rss", tags=["rss"])

app/routers/yle_rss_sv.py

@@ -0,0 +1,141 @@
+import traceback
+import json
+import requests
+import traceback
+import json
+import re
+import feedparser
+import requests
+import logging
+
+from datetime import datetime
+from time import mktime
+from typing import Annotated
+from fastapi import Depends, APIRouter
+
+from settings.defaults import Settings, get_settings
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+@router.get("/yle_sv", summary="Svenska Yle RSS")
+async def update(settings: Annotated[Settings, Depends(get_settings)]):
+
+    feed_url = settings.feeds['yle_sv']['url']
+
+    mastodon_server = settings.mastodon_server
+    mastodon_aid = settings.feeds['yle_sv']['account_id']
+    mastodon_token = settings.feeds['yle_sv']['token']
+    mastodon_get_statuses_url=f'{mastodon_server}/api/v1/accounts/{mastodon_aid}/statuses'
+    mastodon_post_statuses_url=f'{mastodon_server}/api/v1/statuses'
+
+    try:
+        last_status_timestamp=datetime.fromisoformat(load_last_status(mastodon_get_statuses_url, mastodon_token)['created_at'])
+        new_entries=load_feed_rss(feed_url, last_status_timestamp)
+        logger.info(f'Found {len(new_entries)} new entries since {last_status_timestamp}')
+
+        if (len(new_entries) == 0):
+            return {
+                "status": 200,
+                "body": {
+                    "posted_entries": 0,
+                    "successful": True
+                }
+            }
+
+        posted_entries=list(map(lambda x: post_rss_entry_to_mastodon(mastodon_post_statuses_url, mastodon_token, x), new_entries))
+
+        return {
+            "status": 200,
+            "body": {
+                "posted_entries": len(posted_entries),
+                "successful": True
+            }
+        }
+    except Exception as e:
+        msg = ''.join(traceback.format_exception_only(e))
+        logger.error(msg)
+        return {
+            "status": 501,
+            "body": {
+                "posted_entries": 0,
+                "message": msg,
+                "successful": False
+            }
+        }
+
+def split(arr, char):
+    return [tag for subtags in (map(lambda str: str.split(char), arr)) for tag in subtags]
+
+def capitalize(arr, char):
+    result = map(lambda str: str.split(char), arr)
+    result = map(lambda subtag: map(lambda str: str.capitalize(), subtag), result)
+    result = map(lambda subtag: ''.join(subtag), result)
+    return result
+
+def load_last_status(url, token):
+    response=requests.get(url + '?limit=1', headers={ 'Authorization' : f'Bearer {token}' })
+    if response.status_code != 200:
+        raise Exception('Failed to contact Mastodon', response.text)
+
+    body = json.loads(response.text)
+    if len(body) == 0:
+        return json.loads('{ "created_at": "2000-01-01 00:00:00"}')
+
+    return body[0]
+
+
+def post_rss_entry_to_mastodon(url, token, entry):
+
+    title = entry.title
+    description = entry.summary
+    link = entry.link
+
+    linkEnd = entry.link.find('?')
+    if linkEnd > -1:
+        link = entry.link[0:linkEnd]
+    else:
+        link = entry.link
+
+    if 'tags' in entry:
+        categories = [t.get('term') for t in entry.tags]
+        categories = split(categories, ',')
+        categories = capitalize(categories, ' ')
+        categories = capitalize(categories, '–')
+        categories = capitalize(categories, '-')
+        categories = capitalize(categories, '/')
+        categories = capitalize(categories, '\\')
+        categories = map(lambda str: re.sub(r'\s+','', str), categories)
+        categories = map(lambda str: re.sub(r'[0-9.()]+','', str), categories)
+        categories = map(lambda str: re.sub('&','och', str), categories)
+       
+        categories = [str for str in categories if len(str) >= 3]
+        if len(categories) > 0:
+            categories = map(lambda str: str if str.startswith('#') else f'#{str}', categories)
+            categories = ' '.join(categories)
+            message = f"{title}\n\n{description}\n\n{link}\n\n{categories}"
+        else:
+            message = f"{title}\n\n{description}\n\n{link}"
+    else:
+        message = f"{title}\n\n{description}\n\n{link}"
+
+    headers = {
+        'Authorization': f'Bearer {token}',
+        'Content-type': 'application/x-www-form-urlencoded',
+        'User-Agent': 'Serverless Feed'
+    }
+
+    params = {
+        'status': message,
+        'language': 'sv',
+        'visibility': 'public'
+    }
+
+    response = requests.post(url, data=params, headers=headers)
+    if response.status_code != 200:
+        logger.error('Failed to post message', response)
+    return response
+
+def load_feed_rss(url, since):
+    feed=feedparser.parse(url)
+    return [entry for entry in feed.entries if datetime.fromtimestamp(mktime(entry.published_parsed)) > since.replace(tzinfo=datetime.fromtimestamp(mktime(entry.published_parsed)).tzinfo)]

app/settings/defaults.py

@@ -4,6 +4,9 @@ from functools import lru_cache
 class Settings(BaseSettings):
     mastodon_server:str
     openai_api_key:str
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    aws_endpoint_url_s3: str
     feeds: dict[str, dict[str,object]]
     model_config = SettingsConfigDict(env_file=".env", env_nested_delimiter='__', arbitrary_types_allowed=True)
     version:str

compose.yml

@@ -12,7 +12,7 @@ services:
     - 8000:8000
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ./app
           target: /code/app
         - action: rebuild