Merge pull request #12 from OWASP/dev

Dev Merge and add create GH action workflow

.github/workflows/dev-push.yml

@@ -0,0 +1,54 @@
+name: "Dev Release: Build and Push OWASP OFFAT Docker Images to DockerHub"
+
+on:
+  push:
+    branches:
+      - "dev"
+
+jobs:
+  build-and-push-dev-docker-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Branch Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push offat-base docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/base-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-base:dev
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/dev/cli-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat:dev
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat-api docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/dev/backend-api-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-api:dev
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat-api-worker docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/dev/backend-api-worker-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-api-worker:dev
+          platforms: linux/amd64,linux/arm64

.github/workflows/pypi-publish.yml

@@ -0,0 +1,44 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload OWASP OFFAT Python Package to PyPi
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+    - name: change cwd to src directory 
+      run: cd src
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: |
+        cd src
+        python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        packages_dir: src/dist

.github/workflows/release-push.yml

@@ -0,0 +1,53 @@
+name: "Release: Build and Push OWASP OFFAT Docker Images to DockerHub"
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-and-push-main-docker-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Branch Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push offat-base docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/base-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-base:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/main/cli-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat-api docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/main/backend-api-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-api:latest
+          platforms: linux/amd64,linux/arm64
+      - name: Build and push offat-api-worker docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./src/
+          file: ./src/DockerFiles/main/backend-api-worker-Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKERHUB_USERNAME }}/offat-api-worker:latest
+          platforms: linux/amd64,linux/arm64

src/DockerFiles/dev/backend-api-Dockerfile

@@ -0,0 +1,5 @@
+FROM dmdhrumilmistry/offat-base:dev
+
+EXPOSE 8000
+
+ENTRYPOINT [ "python", "-m", "offat.api" ]

src/DockerFiles/dev/backend-api-worker-Dockerfile

@@ -0,0 +1,5 @@
+FROM dmdhrumilmistry/offat-base:dev
+
+WORKDIR /offat
+
+ENTRYPOINT [ "rq", "worker", "offat_task_queue" ]

src/DockerFiles/dev/cli-Dockerfile

@@ -0,0 +1,3 @@
+FROM dmdhrumilmistry/offat-base:dev
+
+ENTRYPOINT [ "offat" ]

src/README.md

@@ -25,6 +25,7 @@ Automatically Tests for vulnerabilities after generating tests from openapi spec
 - User Config Based Testing
 - API for Automating tests and Integrating Tool with other platforms/tools
 - CLI tool
+- Proxy Support
 - Dockerized Project for Easy Usage
 - Open Source Tool with MIT License
 
@@ -164,6 +165,12 @@ The disclaimer advises users to use the open-source project for ethical and legi
 
   > `rl`: requests rate limit, `dr`: delay between requests
 
+- Use along with proxy
+
+```bash
+offat -f swagger_file.json -p http://localhost:8080 --no-ssl -o output.json
+```
+
 - Use user provided inputs for generating tests
 
   ```bash

src/offat/__main__.py

@@ -40,6 +40,8 @@ def start():
     parser.add_argument('-o', '--output', dest='output_file', type=str, help='path to store test results in json format', required=False, default=None)
     parser.add_argument('-H', '--headers', dest='headers', type=str, help='HTTP requests headers that should be sent during testing eg: User-Agent: offat', required=False, default=None, action='append', nargs='*')
     parser.add_argument('-tdc','--test-data-config', dest='test_data_config',help='YAML file containing user test data for tests', required=False, type=str)
+    parser.add_argument('-p', '--proxy', dest='proxy', help='Proxy server URL to route HTTP requests through (e.g., "http://proxyserver:port")', required=False, type=str)
+    parser.add_argument('-ns', '--no-ssl', dest='no_ssl', help='Ignores SSL verification when enabled', action='store_true', required=False) # False -> ignore SSL, True -> enforce SSL check
     args = parser.parse_args()
 
 
@@ -72,6 +74,8 @@ def start():
         rate_limit=rate_limit,
         delay=delay_rate,
         test_data_config=test_data_config,
+        proxy=args.proxy,
+        ssl=args.no_ssl,
     )
 
 

src/offat/api/__main__.py

@@ -1,11 +1,13 @@
 from uvicorn import run
 
-
-if __name__ == '__main__':
+def start():
     run(
         app='offat.api.app:app',
         host="0.0.0.0",
         port=8000,
         workers=2,
         reload=True
-    )
+    )
+
+if __name__ == '__main__':
+    start()

src/offat/config_data_handler.py

@@ -40,7 +40,7 @@ def populate_user_data(actor_data:dict, actor_name:str,tests:list[dict]):
         request_headers[header.get('name')] = header.get('value')
 
     for test in tests:
-        # TODO: replace key and value instead of appending
+        #  replace key and value instead of appending
         test['body_params'] += body_params
         test['query_params'] += query_params
         test['path_params'] += path_params

src/offat/http.py

@@ -1,4 +1,4 @@
-from aiohttp import ClientSession, ClientResponse
+from aiohttp import ClientSession, ClientResponse, TCPConnector
 from os import name as os_name
 
 
@@ -11,19 +11,29 @@
 
 class AsyncRequests:
     '''
-    AsyncRequests class helps to send HTTP requests.
+    AsyncRequests class helps to send HTTP requests with rate limiting options.
     '''
 
-    def __init__(self, headers: dict = None) -> None:
+    def __init__(self, rate_limit:int=None, delay:float=None, headers: dict = None, proxy:str = None, ssl:bool=True, allow_redirects: bool=True) -> None:
         '''AsyncRequests class constructor
         
         Args:
+            rate_limit (int): number of concurrent requests at the same time
+            delay (float): delay between consecutive requests
             headers (dict): overrides default headers while sending HTTP requests
+            proxy (str): proxy URL to be used while sending requests
+            ssl (bool): ignores few SSL errors if value is False
         
         Returns:
             None
         '''
+        self._rate_limit = rate_limit
+        self._delay = delay
         self._headers = headers
+        self._proxy = proxy if proxy else None
+        self._ssl = ssl if ssl else None
+        self._allow_redirects = allow_redirects
+
 
     async def request(self, url: str, method: str = 'GET', session: ClientSession = None, *args, **kwargs) -> ClientResponse:
         '''Send HTTP requests asynchronously
@@ -33,31 +43,34 @@ async def request(self, url: str, method: str = 'GET', session: ClientSession =
             method (str): HTTP methods (default: GET) supports GET, POST, 
             PUT, HEAD, OPTIONS, DELETE
             session (aiohttp.ClientSession): aiohttp Client Session for sending requests
+            
         
         Returns:
             dict: returns request and response data as dict
         '''
         is_new_session = False
+        connector = TCPConnector(ssl=self._ssl,limit=self._rate_limit,)
+
         if not session:
-            session = ClientSession(headers=self._headers)
+            session = ClientSession(headers=self._headers, connector=connector)
             is_new_session = True
 
         method = str(method).upper()
         match method:
             case 'GET':
-                sent_req = session.get(url, *args, **kwargs)
+                sent_req = session.get(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'POST':
-                sent_req = session.post(url, *args, **kwargs)
+                sent_req = session.post(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'PUT':
-                sent_req = session.put(url, *args, **kwargs)
+                sent_req = session.put(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'PATCH':
-                sent_req = session.patch(url, *args, **kwargs)
+                sent_req = session.patch(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'HEAD':
-                sent_req = session.head(url, *args, **kwargs)
+                sent_req = session.head(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'OPTIONS':
-                sent_req = session.options(url, *args, **kwargs)
+                sent_req = session.options(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
             case 'DELETE':
-                sent_req = session.delete(url, *args, **kwargs)
+                sent_req = session.delete(url, proxy=self._proxy, allow_redirects=self._allow_redirects, *args, **kwargs)
 
         resp_data = None
         async with sent_req as response:
@@ -76,45 +89,7 @@ async def request(self, url: str, method: str = 'GET', session: ClientSession =
                 await session.close()
                 del session
 
-        return resp_data
-
-
-class AsyncRLRequests(AsyncRequests):
-    '''
-    Send Asynchronous rate limited HTTP requests.
-    '''
-
-    def __init__(self, rate_limit: int = 20, delay: float = 0.05, headers: dict = None) -> None:
-        '''AsyncRLRequests constructor
-
-        Args:
-            rate_limit (int): number of concurrent requests at the same time
-            delay (float): delay between consecutive requests
-            headers (dict): overrides default headers while sending HTTP requests
-
-        Returns:
-            None
-        '''
-        assert isinstance(delay, float) or isinstance(delay, int)
-        assert isinstance(rate_limit, float) or isinstance(rate_limit, int)
-
-        self._delay = delay
-        self._semaphore = asyncio.Semaphore(rate_limit)
-        super().__init__(headers)
-
-    async def request(self, url: str, method: str = 'GET', session: ClientSession = None, *args, **kwargs) -> ClientResponse:
-        '''Send HTTP requests asynchronously with rate limit and delay between the requests
-
-        Args:
-            url (str): URL of the webpage/endpoint
-            method (str): HTTP methods (default: GET) supports GET, POST, 
-            PUT, HEAD, OPTIONS, DELETE
-            session (aiohttp.ClientSession): aiohttp Client Session for sending requests
-        
-        Returns:
-            dict: returns request and response data as dict
-        '''
-        async with self._semaphore:
-            response = await super().request(url, method, session, *args, **kwargs)
+        if self._delay:
             await asyncio.sleep(self._delay)
-            return response
+
+        return resp_data

src/offat/openapi.py

@@ -19,7 +19,8 @@ def __init__(self, fpath:str, spec:dict=None) -> None:
             self._spec = spec
 
         self.host = self._spec.get('host')
-        self.base_url = f"http://{self.host}{self._spec.get('basePath','')}"
+        self.http_scheme = 'https' if 'https' in self._spec.get('schemes') else 'http'
+        self.base_url = f"{self.http_scheme}://{self.host}{self._spec.get('basePath','')}"
         self.request_response_params = self._get_request_response_params()