fix pipeline about lib #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ETL CI/CD Pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'README.md' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'src/**' | |
| - 'tests/**' | |
| - 'docker-compose.yml' | |
| - '**/*.py' | |
| - '**/*.sql' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the code | |
| uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Create virtual environment | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| - name: Install dependencies | |
| run: | | |
| source venv/bin/activate | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Install Docker and Docker Compose | |
| run: | | |
| # Remove any existing containerd or docker packages | |
| sudo apt-get remove -y containerd containerd.io | |
| # Update the apt package index | |
| sudo apt-get update | |
| # Install Docker | |
| sudo apt-get install -y docker.io | |
| # Install Docker Compose | |
| sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Build and start Docker containers with docker-compose | |
| run: | | |
| docker-compose -f docker-compose.yml up -d | |
| sleep 10 # Wait for the DB to start properly (adjust if needed) | |
| - name: Run database setup | |
| run: | | |
| source venv/bin/activate | |
| python sql/sqlite_db/setup_db.py | |
| - name: Set execute permission for load_data.py | |
| run: | | |
| chmod +x tests/load_data.py | |
| - name: Load data into database | |
| run: | | |
| source venv/bin/activate | |
| python tests/load_data.py | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| pytest tests/test_etl.py | |
| continue-on-error: true | |
| - name: Clean up Docker containers | |
| run: | | |
| docker-compose -f docker-compose.yml down |