import json
import re

def is_english(text):
    vietnamese_diacritics = re.compile(r'[àáảãạăằắẳẵặâầấẩẫậèéẻẽẹêềếểễệìíỉĩịòóỏõọôồốổỗộơờớởỡợùúủũụưừứửữựỳýỷỹỵđÀÁẢÃẠĂẰẮẲẴẶÂẦẤẨẪẬÈÉẺẼẸÊỀẾỂỄỆÌÍỈĨỊÒÓỎÕỌÔỒỐỔỖỘƠỜỚỞỠỢÙÚỦŨỤƯỪỨỬỮỰỲÝỶỸỴĐ]')
    if not vietnamese_diacritics.search(text):
        return True
    
    words = text.lower().split()
    en_stopwords = {'the', 'of', 'and', 'to', 'in', 'is', 'that', 'it', 'for', 'on', 'with', 'as', 'this', 'by', 'are', 'an', 'be', 'at', 'from', 'which'}
    en_count = sum(1 for w in words if w in en_stopwords)
    if en_count > 2 or (len(words) > 0 and en_count / len(words) > 0.05):
        return True
    return False

def main():
    with open('/opt/ai-os/products/ceo/parsed_sources.json', 'r', encoding='utf-8') as f:
        sources = json.load(f)

    en_sources = []
    vn_sources = []

    for i, s in enumerate(sources):
        summary = s.get('summary_en', '')
        title = s.get('title', '')
        if is_english(summary):
            en_sources.append((i, title, summary))
        else:
            vn_sources.append((i, title, summary))

    print(f"Total sources: {len(sources)}")
    print(f"English summaries count: {len(en_sources)}")
    print(f"Vietnamese summaries count: {len(vn_sources)}")
    print("\n--- English Summaries Found: ---")
    for idx, title, text in en_sources:
        print(f"[{idx}] {title}")
        print(f"    Summary: {text[:200]}...")

if __name__ == '__main__':
    main()