import urllib.request
import json
from bs4 import BeautifulSoup

urls = [
    'https://www.greenleafvietnam.com/news/kich-thuoc-xe-7-cho-la-bao-nhieu.html',
    'https://www.greenleafvietnam.com/bang-gia/thue-xe-thang.html',
    'https://www.greenleafvietnam.com/news/mau-hop-dong-thue-xe-thang-dai-han-thu-tuc-nhanh-chong.html',
    'https://www.greenleafvietnam.com/dia-diem-thue-xe/cho-thue-xe-dong-nai',
    'https://www.greenleafvietnam.com/news/cho-thue-xe-16-cho-quan-1-va-cac-khu-vuc-o-sai-gon.html'
]

results = {}
for url in urls:
    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'})
    try:
        response = urllib.request.urlopen(req)
        html = response.read()
        soup = BeautifulSoup(html, 'html.parser')
        
        title = soup.title.string.strip() if soup.title else ''
        meta_desc = soup.find('meta', attrs={'name': 'description'})
        meta_desc_content = meta_desc['content'].strip() if meta_desc and 'content' in meta_desc.attrs else ''
        
        h1 = [h.text.strip() for h in soup.find_all('h1') if h.text.strip()]
        h2 = [h.text.strip() for h in soup.find_all('h2') if h.text.strip()]
        
        paragraphs = [p.text.strip() for p in soup.find_all('p') if p.text.strip()]
        word_count = sum(len(p.split()) for p in paragraphs)
        
        results[url] = {
            'title': title,
            'meta_description': meta_desc_content,
            'h1': h1,
            'h2': h2,
            'word_count': word_count
        }
    except Exception as e:
        results[url] = {'error': str(e)}

with open('public/onpage_data.json', 'w', encoding='utf-8') as f:
    json.dump(results, f, ensure_ascii=False, indent=2)

print("Data saved to public/onpage_data.json")