name: Data Pipeline CI

"on":
  push:
    branches: [ main, master ]
  pull_request:
    branches: [ main, master ]

jobs:
  python-validation:
    name: Python ETL Validation (Pandas)
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
        
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r tasks/01_data_ingestion_transformation/requirements.txt
        pip install -r tasks/01_data_ingestion_transformation/requirements-dev.txt
        
    - name: Lint with Ruff
      run: |
        cd tasks/01_data_ingestion_transformation
        ruff check src/ tests/
        
    - name: Run Python Unit Tests (Pandas)
      run: |
        cd tasks/01_data_ingestion_transformation
        export PYTHONPATH=$PYTHONPATH:$(pwd)
        pytest tests/test_etl.py tests/test_integration.py -v --tb=short

  pyspark-validation:
    name: PySpark ETL Validation
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    
    - name: Set up Java (required for PySpark)
      uses: actions/setup-java@v3
      with:
        distribution: 'temurin'
        java-version: '17'
    
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r tasks/01_data_ingestion_transformation/requirements.txt
        pip install -r tasks/01_data_ingestion_transformation/requirements-dev.txt
        
    - name: Lint with Ruff (Spark modules)
      run: |
        cd tasks/01_data_ingestion_transformation
        ruff check src/etl/*_spark.py tests/test_etl_spark.py
        
    - name: Run PySpark Unit Tests
      run: |
        cd tasks/01_data_ingestion_transformation
        export PYTHONPATH=$PYTHONPATH:$(pwd)
        export JAVA_HOME=$JAVA_HOME
        pytest tests/test_etl_spark.py -v --tb=short
  
  sql-validation:
    name: SQL Query Validation
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
        
    - name: Install SQL test dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r tasks/03_sql/requirements.txt
        
    - name: Lint SQL with sqlfluff
      run: |
        cd tasks/03_sql
        sqlfluff lint balance_history_2024_q1.sql --dialect postgres
        
    - name: Run SQL Unit Tests
      run: |
        cd tasks/03_sql
        pytest tests/ -v --tb=short

  integration-tests:
    name: Integration Tests (Real Infrastructure)
    runs-on: ubuntu-latest
    services:
      minio:
        image: minio/minio:latest
        ports:
          - 9000:9000
          - 9001:9001
        env:
          MINIO_ROOT_USER: minioadmin
          MINIO_ROOT_PASSWORD: minioadmin
        options: >-
          --health-cmd "curl -f http://localhost:9000/minio/health/live || exit 1"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v3
      
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      
      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r tasks/01_data_ingestion_transformation/requirements.txt
          pip install -r tasks/01_data_ingestion_transformation/requirements-dev.txt
      
      - name: Wait for MinIO
        run: |
          timeout 60 bash -c 'until curl -f http://localhost:9000/minio/health/live; do sleep 2; done'
          echo "✅ MinIO is ready"
      
      - name: Run Integration Tests
        env:
          S3_ENDPOINT_URL: http://localhost:9000
          AWS_ACCESS_KEY_ID: minioadmin
          AWS_SECRET_ACCESS_KEY: minioadmin
          TEST_BRONZE_BUCKET: test-bronze
          TEST_SILVER_BUCKET: test-silver
          TEST_QUARANTINE_BUCKET: test-quarantine
          TEST_CONDEMNED_BUCKET: test-condemned
        run: |
          cd tasks/01_data_ingestion_transformation
          export PYTHONPATH=$PYTHONPATH:$(pwd)
          pytest tests/test_s3_integration.py tests/test_idempotency.py tests/test_failure_modes.py tests/test_resilience.py -v -m "integration"
