diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..7cfd551 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,49 @@ +# AGENTS.md + +## Run +``` +python main.py +``` +Uses `runpy` to exec `bot.py`. No tests, no lint, no CI. + +## Architecture +- `bot.py` — Telegram bot (python-telegram-bot), single-worker `asyncio.Queue`, all crawl/parse serialized +- `agent.py` — pydantic-ai with local Ollama (`granite4.1:8b`), three agents (opportunity/event/workshop), `output_type=str` +- `database.py` — PocketBase client, collections `events` / `opportunities` / `workshops` +- `scraper.py` — crawl4ai `AsyncWebCrawler` URL → markdown +- `prompts.py` — system prompts for both agent types +- `schemas.py` — **unused at runtime**, per file comment: "File doesn't do anything, its just an outline" + +## Windows event-loop quirk +DO NOT remove the dual-event-loop pattern in `bot.py`: + +1. PTB runs on `WindowsSelectorEventLoopPolicy` (line 357) +2. Scraper runs in a **separate thread** with `WindowsProactorEventLoopPolicy` (line 25) — required for Playwright subprocesses on Windows + +The `get_clean_content()` wrapper in `bot.py` runs `_run_scraper_in_thread` via `ThreadPoolExecutor`. Scraper functions must not be called directly. + +## Agent output format +Agents use `output_type=str` (not structured output). JSON is parsed manually from `agent.run()` result: +- LLM returns JSON wrapped in `````json ... ````` markdown backticks +- `agent.py:56` strips the backtick wrappers before `json.loads()` +- The markdown wrapper is specified in `prompts.py` — don't change prompt format without updating the strip logic + +## Date formats +- Prompts instruct LLM: `DD-MM-YYYY` (or `DD-MM-YYYY (HH:MM)` for events) +- PocketBase expects: `YYYY-MM-DD HH:MM:SS` +- `database.py` field mapping: event `date_time` → PocketBase `datetime`, `end_date` → `end_datetime`; workshop same; opportunity `deadline` stays `deadline` + +## PocketBase collections +- `events` — fields: `title`, `org`, `datetime`, `end_datetime`, `summary`, `location`, `url` +- `opportunities` — fields: `title`, `org`, `type`, `deadline`, `summary`, `location`, `url` +- `workshops` — fields: `title`, `org`, `datetime`, `end_datetime`, `summary`, `location`, `url` +- Auth uses admin credentials from `.env` + +## Env vars +`.env` required: `TG_TOKEN`, `OLLAMA_BASE_URL`, `ALLOWED_USERS`, `POCKETBASE_URL`, `POCKETBASE_ADMIN_EMAIL`, `POCKETBASE_ADMIN_PASSWORD` + +## Auth +`ALLOWED_USERS` — comma-separated Telegram user IDs (no brackets). Checked on every command/message handler. + +## Save flow +When source is pasted text (not URL), the bot asks for a source URL before saving. `context.user_data` keys: `last_extracted`, `last_source_value`, `last_source_kind`, `last_entry_type`, `awaiting_save_url`, `pending_save_url`, `pending_url_to_process`. diff --git a/README.md b/README.md index ed0d550..85ee7c1 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ POCKETBASE_ADMIN_PASSWORD=secret 4. Set `OLLAMA_BASE_URL` in your `.env` to point to the running API, for example: ```text - OLLAMA_BASE_URL=http://localhost:11434/v1 + OLLAMA_BASE_URL=http://localhost:11434 ``` 5. Verify the API is reachable (example curl): diff --git a/__pycache__/agent.cpython-312.pyc b/__pycache__/agent.cpython-312.pyc index 59a6e04..1678067 100644 Binary files a/__pycache__/agent.cpython-312.pyc and b/__pycache__/agent.cpython-312.pyc differ diff --git a/__pycache__/database.cpython-312.pyc b/__pycache__/database.cpython-312.pyc index f9c3d3c..f242cd6 100644 Binary files a/__pycache__/database.cpython-312.pyc and b/__pycache__/database.cpython-312.pyc differ diff --git a/__pycache__/prompts.cpython-312.pyc b/__pycache__/prompts.cpython-312.pyc index 0e1a3b3..b2d2493 100644 Binary files a/__pycache__/prompts.cpython-312.pyc and b/__pycache__/prompts.cpython-312.pyc differ diff --git a/__pycache__/schemas.cpython-312.pyc b/__pycache__/schemas.cpython-312.pyc index 58410ba..8cada13 100644 Binary files a/__pycache__/schemas.cpython-312.pyc and b/__pycache__/schemas.cpython-312.pyc differ diff --git a/agent.py b/agent.py index 3deb622..b1a2900 100644 --- a/agent.py +++ b/agent.py @@ -4,9 +4,9 @@ from pydantic_ai.providers.ollama import OllamaProvider from dotenv import load_dotenv import os import json -from prompts import OPPORTUNITY_PROMPT, EVENT_PROMPT +from prompts import OPPORTUNITY_PROMPT, EVENT_PROMPT, WORKSHOP_PROMPT -load_dotenv() +load_dotenv(override=True) ollama_url = os.getenv("OLLAMA_BASE_URL") @@ -34,6 +34,14 @@ event_agent = Agent( retries=5 ) +# --- WORKSHOP AGENT --- +workshop_agent = Agent( + model, + output_type=str, + system_prompt=WORKSHOP_PROMPT, + retries=5 +) + async def parse_page(content: str, entry_type: str = "opportunity"): """ Parse content and extract entry data based on type. @@ -43,7 +51,12 @@ async def parse_page(content: str, entry_type: str = "opportunity"): entry_type: Either 'opportunity' or 'event' """ # Select the appropriate agent - agent = opportunity_agent if entry_type == "opportunity" else event_agent + if entry_type == "opportunity": + agent = opportunity_agent + elif entry_type == "event": + agent = event_agent + else: + agent = workshop_agent # 1. Run the agent (which returns a string) print(f"[DEBUG] Generating {entry_type}...") diff --git a/bot.py b/bot.py index 10dd58b..7fb3acd 100644 --- a/bot.py +++ b/bot.py @@ -37,7 +37,7 @@ async def get_clean_content(url: str) -> str: result = await loop.run_in_executor(pool, _run_scraper_in_thread, url) return result -load_dotenv() +load_dotenv(override=True) logging.getLogger("httpx").setLevel(logging.WARNING) # Configuration @@ -103,20 +103,46 @@ def build_url_choice_keyboard(url: str): return InlineKeyboardMarkup([ [InlineKeyboardButton("📅 Process as Event", callback_data='choose_type:event')], [InlineKeyboardButton("📋 Process as Opportunity", callback_data='choose_type:opportunity')], + [InlineKeyboardButton("🛠 Process as Workshop", callback_data='choose_type:workshop')], ]) def build_entry_summary(data, entry_type, saved=False): if entry_type == "event": event_datetime = data.get('date_time') or data.get('datetime') - return ( - f"✅ **{data.get('title', 'Unknown')}**\n" - f"🦆 Org/s: {data.get('org')}\n" - f"📅 Date & Time: {event_datetime}\n" - f"📍 Location: {data.get('location')}\n" - f"🐊 Summary: {data.get('summary')}" - + ("\n\n💾 **Saved to PocketBase!**" if saved else "") - ) + end_date = data.get('end_date') or data.get('end_datetime') + lines = [ + f"✅ **{data.get('title', 'Unknown')}**", + f"🦆 Org/s: {data.get('org')}", + f"📅 Date & Time: {event_datetime}", + ] + if end_date and end_date != 'N/A': + lines.append(f"📅 End Date: {end_date}") + lines.extend([ + f"📍 Location: {data.get('location')}", + f"🐊 Summary: {data.get('summary')}", + ]) + if saved: + lines.append("\n💾 **Saved to PocketBase!**") + return "\n".join(lines) + + if entry_type == "workshop": + event_datetime = data.get('date_time') or data.get('datetime') + end_date = data.get('end_date') or data.get('end_datetime') + lines = [ + f"✅ **{data.get('title', 'Unknown')}**", + f"🦆 Org/s: {data.get('org')}", + f"📅 Date & Time: {event_datetime}", + ] + if end_date and end_date != 'N/A': + lines.append(f"📅 End Date: {end_date}") + lines.extend([ + f"📍 Location: {data.get('location')}", + f"🐊 Summary: {data.get('summary')}", + ]) + if saved: + lines.append("\n💾 **Saved to PocketBase!**") + return "\n".join(lines) return ( f"✅ **{data.get('title', 'Unknown')}**\n" @@ -194,8 +220,9 @@ async def start(update: Update, context: ContextTypes.DEFAULT_TYPE): "Welcome! I can extract arts opportunities and events.\n\n" "📋 **Commands:**\n" "/op - Extract an opportunity\n" - "/ev - Extract an event\n\n" - "You can also send a URL directly and I will ask whether to process it as an event or opportunity." + "/ev - Extract an event\n" + "/ws - Extract a workshop\n\n" + "You can also send a URL directly and I will ask whether to process it as an event, workshop, or opportunity." ) async def handle_opportunity(update: Update, context: ContextTypes.DEFAULT_TYPE): @@ -238,6 +265,26 @@ async def handle_event(update: Update, context: ContextTypes.DEFAULT_TYPE): await update.message.reply_text("📥 Link queued for processing...") await task_queue.put((update, context, input_text, "event", source_kind)) +async def handle_workshop(update: Update, context: ContextTypes.DEFAULT_TYPE): + user_id = update.effective_user.id + if user_id not in ALLOWED_IDS: + await update.message.reply_text("Unauthorized. User ID needs to be added!") + return + + if not context.args: + await update.message.reply_text("Please provide a URL or paste text. Usage: /ws ") + return + + input_text = " ".join(context.args).strip() + if not input_text: + await update.message.reply_text("Please provide a URL or paste text. Usage: /ws ") + return + + source_kind = "url" if input_text.startswith("http") else "text" + + await update.message.reply_text("📥 Link queued for processing...") + await task_queue.put((update, context, input_text, "workshop", source_kind)) + async def handle_followup_text(update: Update, context: ContextTypes.DEFAULT_TYPE): if update.effective_user.id not in ALLOWED_IDS: return @@ -327,6 +374,7 @@ async def _main(): application.add_handler(CommandHandler("start", start)) application.add_handler(CommandHandler("op", handle_opportunity)) application.add_handler(CommandHandler("ev", handle_event)) + application.add_handler(CommandHandler("ws", handle_workshop)) application.add_handler(MessageHandler(filters.TEXT & (~filters.COMMAND), handle_followup_text)) application.add_handler(CallbackQueryHandler(button_handler)) diff --git a/database.py b/database.py index 558d56c..a50fdff 100644 --- a/database.py +++ b/database.py @@ -4,7 +4,7 @@ from pocketbase import PocketBase from schemas import EntrySchema from datetime import datetime -load_dotenv() +load_dotenv(override=True) pb = PocketBase(os.getenv('POCKETBASE_URL')) admin_data = pb.admins.auth_with_password(os.getenv('POCKETBASE_ADMIN_EMAIL'), os.getenv('POCKETBASE_ADMIN_PASSWORD')) @@ -62,23 +62,36 @@ def upload_entry(data, entry_type='opportunity', url=None): try: if entry_type == 'event': - # Map 'date_time' from agent to 'datetime' for PocketBase if 'date_time' in data: original_dt = data['date_time'] - # Convert and map to PocketBase field name data['datetime'] = convert_datetime_to_pocketbase(data['date_time']) - # Remove the original field since PocketBase expects 'datetime' del data['date_time'] if show_debug_msg: print(f"[DEBUG] Event datetime: '{original_dt}' -> '{data['datetime']}'") else: print(f"[WARNING] No 'date_time' field found in event data") - - # Upload to events collection + + if 'end_date' in data: + data['end_datetime'] = convert_datetime_to_pocketbase(data['end_date']) + del data['end_date'] + print(f"[DEBUG] Creating record in 'events' collection with data: {data}") result = pb.collection('events').create(data) print(f"[DEBUG] Successfully created record: {result}") return result + elif entry_type == 'workshop': + if 'date_time' in data: + data['datetime'] = convert_datetime_to_pocketbase(data['date_time']) + del data['date_time'] + + if 'end_date' in data: + data['end_datetime'] = convert_datetime_to_pocketbase(data['end_date']) + del data['end_date'] + + print(f"[DEBUG] Creating record in 'workshops' collection with data: {data}") + result = pb.collection('workshops').create(data) + print(f"[DEBUG] Successfully created record: {result}") + return result else: # Opportunities - convert deadline to datetime format if 'deadline' in data: diff --git a/prompts.py b/prompts.py index 32bc682..4693e29 100644 --- a/prompts.py +++ b/prompts.py @@ -8,7 +8,7 @@ OPPORTUNITY_PROMPT = ( "1. 'title': The title of the opportunity\n" "2. 'org': The name of the organizing body/bodies\n" "3. 'type': The category (e.g., Residency, Funding, Open Call, Workshop).\n" - "4. 'summary': A 3-sentence description of what the opportunity involves.\n" + "4. 'summary': A 3-sentence description of what the opportunity involves, including who it is targeted towards (e.g. emerging artists, students, professionals).\n" "5. 'deadline': The deadline of the opportunity. Format: DD-MM-YYYY. Assume year 2026 if missing.\n" "6. 'location': The physical city/country or 'Online'.\n\n" "# CONSTRAINTS\n" @@ -22,7 +22,7 @@ OPPORTUNITY_PROMPT = ( " \"title\": \"Digital Horizons 2026\",\n" " \"org\": \"Digital Horizons\",\n" " \"type\": \"Residency\",\n" - " \"summary\": \"A residency for digital artists to explore VR. Includes a stipend.\",\n" + " \"summary\": \"A residency for emerging digital artists to explore VR. Open to EU-based practitioners with less than 5 years of professional experience. Includes a stipend.\",\n" " \"deadline\": \"15-11-2026\",\n" " \"location\": \"Berlin, Germany\"\n" "}\n" @@ -36,9 +36,10 @@ EVENT_PROMPT = ( "Analyze the provided text and extract information into these JSON keys:\n" "1. 'title': The name/title of the event\n" "2. 'org': The name of the organizing body/bodies\n" - "3. 'date_time': The date and time of the event. Format: DD-MM-YYYY (HH:MM) or 'N/A' if not specified.\n" - "4. 'summary': A 3-sentence description of what the event is about.\n" - "5. 'location': The physical venue/city/country or 'Online'.\n\n" + "3. 'date_time': The start date and time of the event. Format: DD-MM-YYYY (HH:MM). Assume year 2026 if missing.\n" + "4. 'end_date': The end date and time of the event, if it spans multiple days. Format: DD-MM-YYYY (HH:MM). Use 'N/A' if it is a one-day event or no end date is specified.\n" + "5. 'summary': A 3-sentence description of what the event is about, including the intended audience (e.g. general public, industry professionals, students).\n" + "6. 'location': The physical venue/city/country or 'Online'.\n\n" "# CONSTRAINTS\n" "- Return ONLY the JSON object inside markdown backticks (```json ... ```).\n" "- Do NOT include any introductory or conversational text.\n" @@ -49,8 +50,37 @@ EVENT_PROMPT = ( " \"title\": \"Digital Arts Symposium 2026\",\n" " \"org\": \"Digital Arts Society\",\n" " \"date_time\": \"20-06-2026 14:00\",\n" - " \"summary\": \"Join us for a day of talks and workshops exploring digital art. Meet artists and curators. Includes lunch and networking.\",\n" + " \"end_date\": \"22-06-2026 18:00\",\n" + " \"summary\": \"Join us for a three-day symposium of talks and workshops exploring digital art, aimed at artists, curators, and cultural professionals. Meet leading practitioners in the field. Includes lunch and networking.\",\n" " \"location\": \"London, UK\"\n" "}\n" "```" ) + +WORKSHOP_PROMPT = ( + "You are a precise Data Extraction Specialist. Your goal is to convert " + "unstructured workshop text into a strictly valid JSON object.\n\n" + "# TASK\n" + "Analyze the provided text and extract information into these JSON keys:\n" + "1. 'title': The name/title of the workshop\n" + "2. 'org': The name of the organizing body/bodies\n" + "3. 'date_time': The start date and time of the workshop. Format: DD-MM-YYYY (HH:MM). Assume year 2026 if missing.\n" + "4. 'end_date': The end date and time of the workshop, if it spans multiple days. Format: DD-MM-YYYY (HH:MM). Use 'N/A' if it is a one-day workshop or no end date is specified.\n" + "5. 'summary': A 3-sentence description of what the workshop is about, including the target audience (e.g. beginners, experienced practitioners, students).\n" + "6. 'location': The physical venue/city/country or 'Online'.\n\n" + "# CONSTRAINTS\n" + "- Return ONLY the JSON object inside markdown backticks (```json ... ```).\n" + "- Do NOT include any introductory or conversational text.\n" + "- If a field is missing, use 'N/A'.\n\n" + "# EXAMPLE OUTPUT\n" + "```json\n" + "{\n" + " \"title\": \"Introduction to Digital Sculpture\",\n" + " \"org\": \"Creative Tech Lab\",\n" + " \"date_time\": \"10-09-2026 10:00\",\n" + " \"end_date\": \"12-09-2026 16:00\",\n" + " \"summary\": \"A three-day hands-on workshop covering Blender and 3D printing, tailored for beginners and intermediate digital artists. Participants will create their own digital sculpture. All skill levels welcome.\",\n" + " \"location\": \"Dublin, Ireland\"\n" + "}\n" + "```" +) diff --git a/schemas.py b/schemas.py index 568256b..4a45b6f 100644 --- a/schemas.py +++ b/schemas.py @@ -1,4 +1,4 @@ -from typing import Union, Literal +from typing import Union, Literal, Optional from pydantic import BaseModel, Field ## File doesn't do anything, its just an outline for the schemas @@ -11,6 +11,7 @@ class BaseEntry(BaseModel): class Event(BaseEntry): type: Literal["event"] = "event" date_time: str = Field(description="Date and time of the event") + end_date: Optional[str] = Field(default=None, description="End date and time if multi-day event") location: str = Field(description="Location of the event") class Opportunity(BaseEntry): @@ -18,4 +19,10 @@ class Opportunity(BaseEntry): deadline: str = Field(description="What is the deadline in the format of dd-mm-yy") location: str = Field(description="Location of entry") -EntrySchema = Union[Event, Opportunity] \ No newline at end of file +class Workshop(BaseEntry): + type: Literal["workshop"] = "workshop" + date_time: str = Field(description="Start date and time of the workshop") + end_date: Optional[str] = Field(default=None, description="End date and time if multi-day workshop") + location: str = Field(description="Location of the workshop") + +EntrySchema = Union[Event, Opportunity, Workshop] \ No newline at end of file