import requests
from bs4 import BeautifulSoup
import time
import json

def scrape_simple_job_board(company_url, job_board_path="/careers"):
    """
    A basic scraper for a simple, static career page.
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    
    try:
        response = requests.get(f"{company_url}{job_board_path}", headers=headers, timeout=10)
        response.raise_for_status()
    except requests.exceptions.RequestException as e:
        print(f"Failed to retrieve page: {e}")
        return []

    soup = BeautifulSoup(response.content, 'html.parser')
    # These selectors are hypothetical - YOU MUST INSPECT THE TARGET SITE.
    job_elements = soup.select('div.job-listing a') 
    
    jobs = []
    for job_elem in job_elements[:5]:  # Limit for demo
        job_title = job_elem.text.strip()
        job_url = job_elem.get('href')
        if not job_url.startswith('http'):
            job_url = company_url + job_url
        
        # Now, scrape the individual job posting
        time.sleep(1)  # BE POLITE - critical for scale
        try:
            job_resp = requests.get(job_url, headers=headers, timeout=10)
            job_soup = BeautifulSoup(job_resp.content, 'html.parser')
            # Get the main content - again, inspect the target.
            content_div = job_soup.find('div', {'class': 'job-description'})
            job_content = content_div.get_text(separator='\n', strip=True) if content_div else ""
            
            jobs.append({
                'source_url': job_url,
                'raw_html': str(job_soup),  # Store full HTML for AI processing
                'title': job_title,
                'company': company_url
            })
            print(f"Scraped: {job_title}")
        except Exception as e:
            print(f"Error scraping {job_url}: {e}")
    
    return jobs

# Example usage
if __name__ == "__main__":
    sample_jobs = scrape_simple_job_board("https://exampletechcompany.com")
    # Save raw data for AI processing
    with open('raw_jobs_dump.json', 'w') as f:
        json.dump(sample_jobs, f, indent=2)
    print(f"Scraped {len(sample_jobs)} jobs.")

from langchain.text_splitter import RecursiveCharacterTextSplitter

def preprocess_and_chunk(raw_html, chunk_size=1500, chunk_overlap=200):
    """
    Extract text from HTML and split into manageable chunks for the LLM.
    """
    soup = BeautifulSoup(raw_html, 'html.parser')
    main_text = soup.get_text(separator='\n', strip=True)
    
    # Simple deduplication of lines
    lines = [line for line in main_text.split('\n') if line.strip()]
    unique_lines = []
    seen = set()
    for line in lines:
        if line not in seen:
            seen.add(line)
            unique_lines.append(line)
    clean_text = '\n'.join(unique_lines)
    
    # Chunking for large posts
    if len(clean_text) > chunk_size:
        splitter = RecursiveCharacterTextSplitter(
            chunk_size=chunk_size,
            chunk_overlap=chunk_overlap,
            length_function=len,
            separators=["\n\n", "\n", ".", " ", ""]
        )
        chunks = splitter.split_text(clean_text)
        return chunks
    else:
        return [clean_text]

# Example
with open('raw_jobs_dump.json', 'r') as f:
    raw_jobs = json.load(f)

for job in raw_jobs:
    job['chunks'] = preprocess_and_chunk(job['raw_html'])
    # Don't need raw_html in memory for LLM step; we have chunks.
    del job['raw_html'] 

from pydantic import BaseModel, Field
from typing import List, Optional
from langchain_openai import ChatOpenAI
from langchain.chains import create_extraction_chain_pydantic
import os

# 1. Define your desired schema
class JobPosting(BaseModel):
    job_title: str = Field(description="The official title of the job position.")
    company_name: str = Field(description="The name of the hiring company.")
    salary_range: Optional[str] = Field(description="Estimated or stated salary range, if present.")
    location: Optional[str] = Field(description="City, State, Country, or Remote specification.")
    required_skills: List[str] = Field(description="List of hard skills, technologies, or certifications required (e.g., Python, AWS, PhD).")
    preferred_skills: List[str] = Field(description="List of nice-to-have skills or qualifications.")
    experience_level: Optional[str] = Field(description="e.g., Entry, Mid, Senior, Lead, Director.")
    education_requirements: Optional[str] = Field(description="Required degree or education level.")
    role_summary: Optional[str] = Field(description="A 1-2 sentence summary of the role's core purpose.")

# 2. Set up LLM and chain
llm = ChatOpenAI(
    model="gpt-4o-mini",  # Cost-effective and capable for this task
    temperature=0,  # We want deterministic extraction
    api_key=os.getenv('OPENAI_API_KEY')
)

def extract_job_data_from_chunks(chunks: List[str]) -> JobPosting:
    """
    Processes text chunks through an LLM to extract structured data.
    """
    combined_extraction = {}
    for chunk in chunks:
        # Run extraction on each chunk
        chain = create_extraction_chain_pydantic(pydantic_schema=JobPosting, llm=llm)
        result = chain.invoke(chunk)
        if result and isinstance(result, list) and len(result) > 0:
            chunk_data = result[0].dict()  # Get dict from first (and likely only) Pydantic object
            # Merge logic: for lists (skills), append; for strings, take the first non-empty value.
            for key, value in chunk_data.items():
                if value is None or value == []:
                    continue
                if isinstance(value, list):
                    combined_extraction.setdefault(key, []).extend([v for v in value if v not in combined_extraction.get(key, [])])
                else:
                    if key not in combined_extraction or not combined_extraction[key]:
                        combined_extraction[key] = value
    # Return a JobPosting object from the combined data
    return JobPosting(**combined_extraction) if combined_extraction else None

# 3. Process all scraped jobs
structured_jobs = []
for job in raw_jobs:
    print(f"Processing: {job.get('title')}")
    try:
        extracted_data = extract_job_data_from_chunks(job['chunks'])
        if extracted_data:
            # Add source metadata
            final_data = extracted_data.dict()
            final_data['source_url'] = job['source_url']
            final_data['scraped_title'] = job['title']
            structured_jobs.append(final_data)
            print(f"  Successfully extracted.")
        else:
            print(f"  Failed to extract.")
    except Exception as e:
        print(f"  Error: {e}")

# Save the valuable structured data
with open('structured_jobs.json', 'w') as f:
    json.dump(structured_jobs, f, indent=2)

print(f"\nExtraction Complete: {len(structured_jobs)} / {len(raw_jobs)} jobs successfully processed.")

import pandas as pd
import matplotlib.pyplot as plt
from collections import Counter
import ast

# Load data
df = pd.read_json('structured_jobs.json')

# 1. Top Required Skills across all competitors
all_required_skills = []
for skill_list in df['required_skills'].dropna():
    # Ensure it's a list
    if isinstance(skill_list, str):
        skill_list = ast.literal_eval(skill_list)
    all_required_skills.extend(skill_list)

top_skills = Counter(all_required_skills).most_common(20)
skills_df = pd.DataFrame(top_skills, columns=['Skill', 'Count'])
print("Top 20 Required Skills in the Market:")
print(skills_df)

# 2. Experience Level Distribution
exp_dist = df['experience_level'].value_counts()
print("\nExperience Level Distribution:")
print(exp_dist)

# 3. Salary Range Analysis (if you have enough data)
salary_df = df['salary_range'].dropna()
print(f"\nCollected {len(salary_df)} salary ranges.")

# 4. Company-specific Tech Stack (Example)
target_company = "ExampleTechCompany"
company_df = df[df['company_name'].str.contains(target_company, case=False, na=False)]
company_skills = []
for skill_list in company_df['required_skills'].dropna():
    if isinstance(skill_list, str):
        skill_list = ast.literal_eval(skill_list)
    company_skills.extend(skill_list)

print(f"\n{target_company}'s Top Tech:")
print(Counter(company_skills).most_common(10))

# Visualization
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
skills_df.head(10).plot(kind='barh', x='Skill', y='Count', ax=axes[0], title='Top 10 Required Skills')
exp_dist.plot(kind='pie', ax=axes[1], autopct='%1.1f%%', title='Experience Level Distribution')
plt.tight_layout()
plt.savefig('job_market_insights.png')
plt.show()