""" Market price scraper for CropPilot This module provides tools to scrape agricultural market prices from various sources. """ import httpx import asyncio from bs4 import BeautifulSoup from typing import List, Dict, Any, Optional import json from datetime import datetime import logging # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class MarketScraper: """Base scraper class for market data""" async def fetch_page(self, url: str) -> Optional[str]: """Fetch HTML content from a URL""" try: async with httpx.AsyncClient(timeout=30.0) as client: response = await client.get(url) response.raise_for_status() return response.text except Exception as e: logger.error(f"Error fetching {url}: {e}") return None class AgMarknetScraper(MarketScraper): """Scraper for Agmarknet portal (Indian agricultural markets)""" BASE_URL = "https://agmarknet.gov.in" SEARCH_URL = f"{BASE_URL}/SearchCmmMkt.aspx" async def get_prices(self, commodity: str, state: str = None, district: str = None) -> List[Dict[Any, Any]]: """Fetch prices for a specific commodity across different states in India Note: This is a simplified implementation that would need to be expanded to handle the actual form submission and parsing of the AgMarknet portal. """ try: # In a real implementation, you would: # 1. Fetch the form page # 2. Extract form data and viewstate # 3. Submit form with commodity and optionally state/district # 4. Parse results from multiple markets across India # For demo purposes, we're returning mock data for multiple locations across India today = datetime.now().strftime("%Y-%m-%d") # Create data that resembles what we'd get from scraping major markets across India if commodity.lower() in ["wheat", "gehu", "gahu"]: return [ {"id": 101, "crop": "Wheat (Gehu)", "price_per_quintal": 2450, "location": "Pune Mandi, Maharashtra", "date": today}, {"id": 102, "crop": "Wheat (Premium)", "price_per_quintal": 2600, "location": "Pune Mandi, Maharashtra", "date": today}, {"id": 103, "crop": "Wheat (Gehu)", "price_per_quintal": 2200, "location": "Indore Mandi, Madhya Pradesh", "date": today}, {"id": 104, "crop": "Wheat (Premium)", "price_per_quintal": 2520, "location": "Ludhiana Mandi, Punjab", "date": today}, {"id": 105, "crop": "Wheat (Standard)", "price_per_quintal": 2380, "location": "Karnal Mandi, Haryana", "date": today}, {"id": 106, "crop": "Wheat (Gehu)", "price_per_quintal": 2250, "location": "Kanpur Mandi, Uttar Pradesh", "date": today} ] elif commodity.lower() in ["onion", "kanda", "pyaz"]: return [ {"id": 201, "crop": "Onion (Kanda)", "price_per_quintal": 1350, "location": "Nashik Mandi, Maharashtra", "date": today}, {"id": 202, "crop": "Onion (Red)", "price_per_quintal": 1450, "location": "Nashik Mandi, Maharashtra", "date": today}, {"id": 203, "crop": "Onion (Medium)", "price_per_quintal": 1200, "location": "Ahmedabad Mandi, Gujarat", "date": today}, {"id": 204, "crop": "Onion (Small)", "price_per_quintal": 950, "location": "Bengaluru Mandi, Karnataka", "date": today}, {"id": 205, "crop": "Onion (Large)", "price_per_quintal": 1550, "location": "Azadpur Mandi, Delhi", "date": today}, {"id": 206, "crop": "Onion (Red)", "price_per_quintal": 1280, "location": "Kurnool Mandi, Andhra Pradesh", "date": today} ] elif commodity.lower() in ["soybean", "soyabean"]: return [ {"id": 301, "crop": "Soybean", "price_per_quintal": 4580, "location": "Ujjain Mandi, Madhya Pradesh", "date": today}, {"id": 302, "crop": "Soybean (Yellow)", "price_per_quintal": 4650, "location": "Indore Mandi, Madhya Pradesh", "date": today}, {"id": 303, "crop": "Soybean", "price_per_quintal": 4520, "location": "Akola Mandi, Maharashtra", "date": today}, {"id": 304, "crop": "Soybean", "price_per_quintal": 4490, "location": "Kota Mandi, Rajasthan", "date": today}, {"id": 305, "crop": "Soybean (Black)", "price_per_quintal": 4400, "location": "Davangere Mandi, Karnataka", "date": today} ] elif commodity.lower() in ["rice", "paddy", "chawal"]: return [ {"id": 401, "crop": "Rice (Common)", "price_per_quintal": 3200, "location": "Karnal Mandi, Haryana", "date": today}, {"id": 402, "crop": "Rice (Basmati)", "price_per_quintal": 8500, "location": "Amritsar Mandi, Punjab", "date": today}, {"id": 403, "crop": "Rice (Sona Masuri)", "price_per_quintal": 4200, "location": "Nizamabad Mandi, Telangana", "date": today}, {"id": 404, "crop": "Rice (Ponni)", "price_per_quintal": 3800, "location": "Thanjavur Mandi, Tamil Nadu", "date": today}, {"id": 405, "crop": "Rice (Parboiled)", "price_per_quintal": 3100, "location": "Bardhaman Mandi, West Bengal", "date": today}, {"id": 406, "crop": "Rice (Joha)", "price_per_quintal": 7200, "location": "Jorhat Mandi, Assam", "date": today} ] elif commodity.lower() in ["sugarcane", "sugar cane", "oos"]: return [ {"id": 501, "crop": "Sugarcane", "price_per_tonne": 3200, "location": "Mill Gate Rate, Maharashtra", "date": today}, {"id": 502, "crop": "Sugarcane", "price_per_tonne": 3150, "location": "Mill Gate Rate, Uttar Pradesh", "date": today}, {"id": 503, "crop": "Sugarcane", "price_per_tonne": 3280, "location": "Mill Gate Rate, Karnataka", "date": today}, {"id": 504, "crop": "Sugarcane", "price_per_tonne": 3180, "location": "Mill Gate Rate, Tamil Nadu", "date": today}, {"id": 505, "crop": "Sugarcane", "price_per_tonne": 3220, "location": "Mill Gate Rate, Gujarat", "date": today} ] elif commodity.lower() in ["cotton", "kapas"]: return [ {"id": 601, "crop": "Cotton (Medium Staple)", "price_per_quintal": 6200, "location": "Rajkot Mandi, Gujarat", "date": today}, {"id": 602, "crop": "Cotton (Long Staple)", "price_per_quintal": 6800, "location": "Adilabad Mandi, Telangana", "date": today}, {"id": 603, "crop": "Cotton (J-34)", "price_per_quintal": 6100, "location": "Sirsa Mandi, Haryana", "date": today}, {"id": 604, "crop": "Cotton (Shankar-6)", "price_per_quintal": 6700, "location": "Ahmednagar Mandi, Maharashtra", "date": today}, {"id": 605, "crop": "Cotton (DCH-32)", "price_per_quintal": 7200, "location": "Dharwad Mandi, Karnataka", "date": today} ] else: return [] except Exception as e: logger.error(f"Error fetching prices for {commodity} in {district}, {state}: {e}") return [] class KrishiMandiScraper(MarketScraper): """Scraper for KrishiMandi (another market source)""" BASE_URL = "https://krishimandi.example.com" # Example URL async def get_prices(self, crop: str, state: str = None, district: str = None) -> List[Dict[Any, Any]]: """Get prices from Krishi Mandi across India""" # Implementation would be similar to AgMarknet but would cover all states # This is a placeholder for demonstration return [] # Factory function to get appropriate scraper def get_scraper(source: str = "agmarknet"): """Return appropriate scraper based on source name""" scrapers = { "agmarknet": AgMarknetScraper, "krishimandi": KrishiMandiScraper, } scraper_class = scrapers.get(source.lower()) if not scraper_class: raise ValueError(f"Unknown scraper source: {source}") return scraper_class() async def get_all_prices() -> List[Dict[Any, Any]]: """Get current prices for common crops across all of India from multiple sources""" scraper = get_scraper("agmarknet") # Expanded list of common crops across India common_crops = ["wheat", "onion", "soybean", "sugarcane", "rice", "cotton"] tasks = [scraper.get_prices(crop) for crop in common_crops] results = await asyncio.gather(*tasks) # Flatten the list of lists all_prices = [] for result in results: all_prices.extend(result) return all_prices # When run directly, this will test the scraper if __name__ == "__main__": async def test(): prices = await get_all_prices() print(json.dumps(prices, indent=2)) asyncio.run(test())