ci_cd_pipeline.yml

name: ETL CI/CD Pipeline

on:
  push:
    branches:
      - main
    paths-ignore:
      - 'README.md'

  pull_request:
    branches:
      - main
    paths:
      - 'src/**'
      - 'tests/**'
      - 'docker-compose.yml'
      - '**/*.py'
      - '**/*.sql'

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout the code
        uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.9

      - name: Create virtual environment
        run: |
          python -m venv venv
          source venv/bin/activate

      - name: Install dependencies
        run: |
          source venv/bin/activate
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install Docker and Docker Compose
        run: |
          # Remove any existing containerd or docker packages
          sudo apt-get remove -y containerd containerd.io
          
          # Update the apt package index
          sudo apt-get update

          # Install Docker
          sudo apt-get install -y docker.io

          # Install Docker Compose
          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
          sudo chmod +x /usr/local/bin/docker-compose

      - name: Build and start Docker containers with docker-compose
        run: |
          docker-compose -f docker-compose.yml up -d
          sleep 10  # Wait for the DB to start properly (adjust if needed)

      - name: Run database setup
        run: |
          source venv/bin/activate
          python sql/sqlite_db/setup_db.py

      - name: Set execute permission for load_data.py
        run: |
          chmod +x tests/load_data.py

      - name: Load data into database
        run: |
          source venv/bin/activate
          python tests/load_data.py

      - name: Run tests
        run: |
          source venv/bin/activate
          pytest tests/test_etl.py
        continue-on-error: true

      - name: Clean up Docker containers
        run: |
          docker-compose -f docker-compose.yml down