Policy API Integration (#499)

* move certain top-level directories into "scubagoggles"

* implementation with policy API

* correct anchor to setup utility

* first step in fixing smoke test workflow

* improve setup for smoke test; accommodate AMD64 OPA on ARM64 macOS

* implemented initial feedback from pull request

* argument parser: ensure credentials is a Path

* Complete implementation of service account credentials for policy API

* fix GwsAuth for service accounts - they have no token file
getopa: tolerate missing "v" in specified version

* documentation: moved from with "scubagoggles" subdirectory to top-level directory

* build: create output directory if non-existent

* documentation: minor changes

* setup: download OPA; resolve other PR issues

* smoke_test: remove default value for "subjectemail" in test_scubagoggles_output()

* groups 4.1, 5.1: correct for probable Google changes in enumeration values
drive 1.3, 1.4, 1.5, 1.7: correct misunderstanding of sharing option flags

* drive rego: remove unused import

* commoncontrols 3.1: remove policy API check for employee ID challenge (see 11/6 change)
commoncontrols 4.1: use "friendly" value in non-compliance message

* commoncontrols: remove 9.1, 9.2 policy API implementation

* commoncontrols: check "takeout_service_status" in 12.1

* policy_api: implement policy reduction and apply default values
scuba_argument_parser: fix bug in converting argument value data types
drive rego: fix 6.1 for subOUs/groups
policy_api: add ability to dump Google's raw policy response
installation & OPA instructions: add more detail about setup downloading OPA

.gitattributes

@@ -0,0 +1,23 @@
+# If you know it's a text file, don't use "=auto" because with that you're
+# letting Git decide whether it's text (and YOU already know it IS).
+
+*.css text
+*.csv text
+*.html text
+*.jpg binary
+*.js text
+*.json text
+*.md text
+*.pdf binary
+*.png binary
+*.py text
+*.rego text
+*.sh text eol=lf
+*.svg text
+*.txt text
+*.yaml text
+*.yml text
+.gitattributes text
+.gitignore text
+LICENSE text
+MANIFEST.in text

.github/actions/setup-dependencies-macos/action.yml

@@ -5,7 +5,7 @@ inputs:
     default: "macos"
   opa-version:
     required: true
-    default: "0.60.0"
+    default: "v0.60.0"
   python-version:
     required: true
 
@@ -15,10 +15,9 @@ runs:
     - name: Setup virtualenv
       shell: bash
       run: |
-        pip install virtualenv
-        virtualenv -p python .venv
+        python -m venv .venv
         source .venv/bin/activate
-    
+
     - name: Install dependencies
       shell: bash
       run: |
@@ -30,5 +29,5 @@ runs:
     - name: Download OPA executable
       shell: bash
       run: |
-        python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }}
-        chmod +x opa_darwin_amd64
+        scubagoggles setup -m -nc -nd -d ~/scubagoggles -r ~/scubagoggles -c credentials.json
+        scubagoggles getopa -v ${{ inputs.opa-version }}

.github/actions/setup-dependencies-windows/action.yml

@@ -5,7 +5,7 @@ inputs:
     default: "windows"
   opa-version:
     required: true
-    default: "0.60.0"
+    default: "v0.60.0"
   python-version:
     required: true
 
@@ -15,9 +15,8 @@ runs:
     - name: Setup virtualenv
       shell: powershell
       run: |
-        pip install virtualenv
         python -m venv .venv
-        .venv\Scripts\activate
+        .venv\Scripts\activate.ps1
 
     - name: Install dependencies
       shell: powershell
@@ -29,4 +28,6 @@ runs:
 
     - name: Download OPA executable
       shell: powershell
-      run: python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }}
+      run: |
+        scubagoggles setup -m -nc -nd -d ~/scubagoggles -r ~/scubagoggles -c credentials.json
+        scubagoggles getopa -v ${{ inputs.opa-version }}

.github/workflows/get_opa_hash.yml

@@ -13,6 +13,6 @@ jobs:
         ref: main
     - name: Get OPA Hash
       run: |
-           cd ./repo/utils
+           cd ./repo/scubagoggles/utils
            chmod +x checkOPAHash.sh
            ./checkOPAHash.sh

.github/workflows/pylint.yml

@@ -20,18 +20,18 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.9.2"]
+        python-version: ["3.9.19"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pylint==3.2.7 pytest selenium
+        pip install pylint pytest selenium
     - name: Analysing the code with pylint
       run: |
         pylint -d R0913,R0914,R0915,R1702,W0718,W0719,R0801 $(git ls-files '*.py')

.github/workflows/run_opa_tests.yml

@@ -28,15 +28,15 @@ jobs:
         version: latest
 
     - name: Run OPA Check
-      run: opa check rego Testing/RegoTests --strict
+      run: opa check scubagoggles/rego scubagoggles/Testing/RegoTests --strict
 
     - name: Run OPA Tests
-      run: opa test rego/*.rego Testing/RegoTests/**/*.rego -v
+      run: opa test scubagoggles/rego/*.rego scubagoggles/Testing/RegoTests/**/*.rego -v
 
     - name: Setup Regal
-      uses: StyraInc/setup-regal@v0.2.0
+      uses: StyraInc/setup-regal@v1
       with:
-        version: v0.15.0
+        version: 0.27.0
 
     - name: Run Regal Lint
-      run: regal lint --format github rego Testing
+      run: regal lint --format github scubagoggles/rego scubagoggles/Testing/RegoTests

.github/workflows/run_release.yml

@@ -1,4 +1,4 @@
-on: 
+on:
   workflow_dispatch:
     inputs:
       releaseName:
@@ -14,33 +14,42 @@ name: Build Release
 
 jobs:
   build-and-deploy:
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     env:
+      PYTHON_VERSION: "3.12"
       RELEASE_VERSION: ${{ inputs.version }}
     permissions:
       contents: write
     steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        path: repo
-        ref: main
-    - name: Package Repo for release
-      shell: pwsh
-      run: |
-        $PackageName = "scubagoggles"
-        Remove-Item -Recurse -Force repo -Include .git*
-        Move-Item -Path repo -Destination "${PackageName}-${env:RELEASE_VERSION}" -Force
-        Compress-Archive -Path "${PackageName}-${env:RELEASE_VERSION}" -DestinationPath "${PackageName}-${env:RELEASE_VERSION}.zip"
-        Get-ChildItem -Path . | Write-Output
-    - name: release
-      uses: softprops/action-gh-release@v1
-      id: create_release
-      with:
-        draft: true
-        prerelease: false
-        name: v${{ inputs.releaseName }}
-        tag_name: v${{ inputs.version }}
-        files: scubagoggles-${{ inputs.version }}.zip
-        generate_release_notes: true
-        fail_on_unmatched_files: true
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt
+      - name: Package Repo for release
+        shell: bash
+        run: |
+          reqv="${{ inputs.version }}"
+          reqv="${reqv//[[:space:]]/}"
+          [[ ! "$reqv" =~ ^[0-9]{1,2}(\.[0-9]{1,2}){2}$ ]] \
+            && echo "invalid request version: $reqv" && exit 1
+          sgv=$(grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+' scubagoggles/__init__.py)
+          [[ "$reqv" == "$sgv" ]] \
+            || (echo "version mismatch - current: $sgv" && exit 1)
+          ./scubagoggles/utils/build.sh -r "$PWD" -t "$GITHUB_REF_NAME"
+      - name: release
+        uses: softprops/action-gh-release@v1
+        id: create_release
+        with:
+          draft: true
+          prerelease: false
+          name: v${{ inputs.releaseName }}
+          tag_name: v${{ inputs.version }}
+          files: |
+            scubagoggles-*-py3-none-any.whl
+            scubagoggles-*.tar.gz
+          generate_release_notes: true
+          fail_on_unmatched_files: true

.github/workflows/run_smoke_test.yml

@@ -1,5 +1,5 @@
-name: Run Smoke Test 
-on: 
+name: Run Smoke Test
+on:
   pull_request:
     types: [opened, reopened]
     branches:
@@ -32,7 +32,7 @@ on:
         description: "Choose OPA version"
         required: true
         type: string
-        default: "0.60.0"
+        default: "v0.60.0"
 
 jobs:
   configuration:
@@ -77,33 +77,33 @@ jobs:
 
       - name: Setup Python v${{ matrix.python-version }}
         uses: actions/setup-python@v5
-        with: 
+        with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
           cache-dependency-path: "requirements.txt"
-      
+
       - name: Setup Dependencies (Windows)
         if: ${{ matrix.operating-system == 'windows-latest' }}
         uses: ./.github/actions/setup-dependencies-windows
         with:
           operating-system: "windows"
           opa-version: ${{ needs.configuration.outputs.opa-version }}
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Setup Dependencies (macOS)
         if: ${{ matrix.operating-system == 'macos-latest' }}
         uses: ./.github/actions/setup-dependencies-macos
         with:
           operating-system: "macos"
           opa-version: ${{ needs.configuration.outputs.opa-version }}
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Setup credentials for service account
         id: create-json
         uses: jsdaniell/[email protected]
-        with: 
+        with:
           name: "credentials.json"
           json: ${{ secrets.GWS_GITHUB_AUTOMATION_CREDS }}
 
       - name: Run ScubaGoggles and check for correct output
-        run: pytest ./Testing/Functional/SmokeTests/ -vvv --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --customerdomain="${{ secrets.GWS_DOMAIN }}"
+        run: pytest ./scubagoggles/Testing/Functional/SmokeTests/ -vvv --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --customerdomain="${{ secrets.GWS_DOMAIN }}"