From 4386a5619941cc4a1383fc5493d10c8aab8c6cb1 Mon Sep 17 00:00:00 2001 From: Stefan Schneider <28340802+stefanbschneider@users.noreply.github.com> Date: Thu, 4 May 2023 20:07:13 +0200 Subject: [PATCH] Simplify setup, publish to PyPI, disable LSTM predictor (#211) * Remove unnecessary dependencies * Add MIT license * Use Python 3.7-3.11 for CI * Remove Py 3.10 which is interpreted as Py 3.1 * Copy common-utils into coordsim and disable lstm prediction * Add publish workflow * Add pyyaml dependency * Remove Py 3.11 pipeline --- .github/workflows/python-package.yml | 4 +- .github/workflows/python-publish.yml | 41 +++++ README.md | 5 +- requirements.txt | 2 - setup.py | 31 +++- src/LICENSE | 21 +++ src/animations/__init__.py | 0 src/common/__init__.py | 0 src/common/common_functionalities.py | 106 ++++++++++++ src/dummy_env/__init__.py | 5 + src/dummy_env/dummy_simulator.py | 154 ++++++++++++++++++ src/siminterface/simulator.py | 8 +- src/spinterface/__init__.py | 7 + src/spinterface/spinterface.py | 231 +++++++++++++++++++++++++++ 14 files changed, 597 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/python-publish.yml delete mode 100644 requirements.txt create mode 100644 src/LICENSE create mode 100644 src/animations/__init__.py create mode 100644 src/common/__init__.py create mode 100644 src/common/common_functionalities.py create mode 100644 src/dummy_env/__init__.py create mode 100644 src/dummy_env/dummy_simulator.py create mode 100644 src/spinterface/__init__.py create mode 100644 src/spinterface/spinterface.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 29743de04..98499b656 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6] + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -27,7 +27,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools pip install flake8 nose2 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install . - name: Lint with flake8 run: | flake8 src --count --exit-zero --show-source --statistics diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 000000000..6b67f91ec --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,41 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + \ No newline at end of file diff --git a/README.md b/README.md index 8950d1fd9..90081b4e5 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,11 @@ Feel free to open a pull request and add your own project if you use coord-sim! ## Setup -Requires **Python 3.6** (newer versions do not support the required TF 1.14). Install with (ideally using [virtualenv](https://virtualenv.pypa.io/en/stable/)): +Install with (ideally using [virtualenv](https://virtualenv.pypa.io/en/stable/)): ```bash -pip install -r requirements.txt +pip install . +# For dev install: pip install -e . ``` diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5bfbb60da..000000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ ---editable git+https://github.com/RealVNF/common-utils#egg=common-utils ---editable . \ No newline at end of file diff --git a/setup.py b/setup.py index 64b15f110..6d6fe5aa2 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,26 @@ +import os + from setuptools import setup, find_packages + +# read the contents of the README file +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f: + long_description = f.read() + requirements = [ - 'scipy==1.5.4', 'simpy>=4', 'networkx==2.4', 'geopy', - 'pyyaml>=5.1', 'numpy>=1.16.5,<=1.19.5', - 'common-utils', - 'scikit-learn', 'pandas==1.1.5', - 'tensorflow==1.14.0', - 'keras==2.2.5', 'matplotlib', + 'pyyaml' +] +# extra requirements for the lstm_predictor (usually not needed) +lstm_extra_requirements = [ + "scikit-learn", + 'keras==2.2.5', ] - test_requirements = [ 'flake8', 'nose2' @@ -21,14 +28,16 @@ setup( name='coord-sim', - version='2.1.1', + version='2.2.0', description='Simulate flow-level, inter-node network coordination including scaling and placement of services and ' 'scheduling/balancing traffic between them.', url='https://github.com/RealVNF/coord-sim', author='Stefan Schneider', package_dir={'': 'src'}, packages=find_packages('src'), + python_requires=">=3.6.0", install_requires=requirements, + extras_require={"lstm": lstm_extra_requirements}, tests_require=test_requirements, zip_safe=False, entry_points={ @@ -38,4 +47,10 @@ 'lstm-predict=coordsim.traffic_predictor.lstm_predictor:main' ], }, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + license="MIT", ) diff --git a/src/LICENSE b/src/LICENSE new file mode 100644 index 000000000..1d671e069 --- /dev/null +++ b/src/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Stefan Schneider + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/src/animations/__init__.py b/src/animations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/common/__init__.py b/src/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/common/common_functionalities.py b/src/common/common_functionalities.py new file mode 100644 index 000000000..8cddd3833 --- /dev/null +++ b/src/common/common_functionalities.py @@ -0,0 +1,106 @@ +import numpy as np +import os +import yaml +import networkx as nx +from shutil import copyfile + +# url = 'https://github.com/numpy/numpy/blob/master/numpy/random/mtrand.pyx#L778' +# a threshold for floating point arithmetic error handling +accuracy = np.sqrt(np.finfo(np.float64).eps) + + +def normalize_scheduling_probabilities(input_list: list) -> list: + """ returns a rounded off list with the sum of all elements in the list to be equal to 1.0 + Handles these case: + 1) All the elements of the list are 0 -> the Probabilities are equally distributed + 2) When the sum(input_list) is away from 1.0 by an offset -> each prob. is divided by sum(input_list) and + the difference of the sum of this new list to 1.0 is added to the first element of the list. + 3) An empty list is provided as input -> simply returns an empty list. + Because of [1] an error range of +-0.000000014901161193847656 in the sum has to be handled. + [1]: https://stackoverflow.com/questions/588004/is-floating-point-math-broken + """ + + output_list = [] + # to handle the empty list case, we just return the empty list back + if len(input_list) == 0: + return output_list + + offset = 1 - sum(input_list) + + # a list with all elements 0, will be equally distributed to sum-up to 1. + # sum can also be 0 if some elements of the list are negative. + # In our case the list contains probabilities and they are not supposed to be negative, hence the case won't arise + if sum(input_list) == 0: + output_list = [round(1 / len(input_list), 10)] * len(input_list) + + # Because of floating point precision (.59 + .33 + .08) can be equal to .99999999 + # So we correct the sum only if the absolute difference is more than a tolerance(0.000000014901161193847656) + else: + if abs(offset) > accuracy: + sum_list = sum(input_list) + # we divide each number in the list by the sum of the list, so that Prob. Distribution is approx. 1 + output_list = [round(prob / sum_list, 10) for prob in input_list] + else: + output_list = input_list.copy() + + # 1 - sum(output_list) = the diff. by which the elements of the list are away from 1.0, could be +'ive /-i've + new_offset = 1 - sum(output_list) + if new_offset != 0: + i = 0 + while output_list[i] + new_offset < 0: + i += 1 + # the difference is added/subtracted from the 1st element of the list, which is also rounded to 2 decimal points + output_list[i] = output_list[i] + new_offset + assert abs(1 - sum(output_list)) < accuracy, "Sum of list not equal to 1.0" + return output_list + + +def create_input_file(target_dir, num_ingress, algo): + input_file_loc = f"{target_dir}/input.yaml" + os.makedirs(f"{target_dir}", exist_ok=True) + with open(input_file_loc, "w") as f: + inputs = {"num_ingress": num_ingress, "algorithm": algo} + yaml.dump(inputs, f, default_flow_style=False) + + +def num_ingress(network_path): + no_ingress = 0 + network = nx.read_graphml(network_path, node_type=int) + for node in network.nodes(data=True): + if node[1]["NodeType"] == "Ingress": + no_ingress += 1 + return no_ingress + + +def copy_input_files(target_dir, network_path, service_path, sim_config_path): + """Create the results directory and copy input files""" + new_network_path = f"{target_dir}/{os.path.basename(network_path)}" + new_service_path = f"{target_dir}/{os.path.basename(service_path)}" + new_sim_config_path = f"{target_dir}/{os.path.basename(sim_config_path)}" + + os.makedirs(target_dir, exist_ok=True) + copyfile(network_path, new_network_path) + copyfile(service_path, new_service_path) + copyfile(sim_config_path, new_sim_config_path) + + +def get_ingress_nodes_and_cap(network, cap=False): + """ + Gets a NetworkX DiGraph and returns a list of ingress nodes in the network and the largest capacity of nodes + Parameters: + network: NetworkX Digraph + cap: boolean to return the capacity also if True + Returns: + ing_nodes : a list of Ingress nodes in the Network + node_cap : the single largest capacity of all the nodes of the network + """ + ing_nodes = [] + node_cap = {} + for node in network.nodes(data=True): + node_cap[node[0]] = node[1]['cap'] + if node[1]["type"] == "Ingress": + ing_nodes.append(node[0]) + if cap: + return ing_nodes, node_cap + else: + return ing_nodes diff --git a/src/dummy_env/__init__.py b/src/dummy_env/__init__.py new file mode 100644 index 000000000..5b3a9193e --- /dev/null +++ b/src/dummy_env/__init__.py @@ -0,0 +1,5 @@ +""" Package to containing a dummy implementation for the environment. +""" +from .dummy_simulator import DummySimulator + +__all__ = ['DummySimulator'] diff --git a/src/dummy_env/dummy_simulator.py b/src/dummy_env/dummy_simulator.py new file mode 100644 index 000000000..2b5356171 --- /dev/null +++ b/src/dummy_env/dummy_simulator.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +""" +Dummy simulator for testing purposes. +""" +from spinterface import SimulatorAction, SimulatorInterface, SimulatorState + + +class DummySimulator(SimulatorInterface): + """ + return a mixture of static and random values + """ + + def __init__(self, network_file: str, service_functions_file: str, config_file: str, test_mode=False): + super(DummySimulator, self).__init__(test_mode) + + def init(self, seed: int, + trace=None) -> SimulatorState: + """ returns fixed init state + + Parameters + ---------- + config_file + network_file + service_functions_file + seed + + Returns + ------- + state: SimulatorState + + """ + return self.example_state_1 + + def apply(self, actions: SimulatorAction) -> SimulatorState: + """ returns fixed simulator state + + Parameters + ---------- + actions + + Returns + ------- + state: SimulatorState + + """ + return self.example_state_1 + + def get_active_ingress_nodes(self): + return ['pop0'] + + @property + def example_state_1(self): + """ Returns a fixed instance of a simulator state + + Returns + ------- + state: SimulatorState + + """ + + state = SimulatorState( + network={ + 'nodes': [ + { + 'id': 'pop0', + 'resource': 42.0, + 'used_resources': 0.21 + }, + { + 'id': 'pop1', + 'resource': 42.0, + 'used_resources': 0.21 + }, + { + 'id': 'pop2', + 'resource': 42.0, + 'used_resources': 0.21 + } + ], + 'edges': [ + { + 'src': 'pop0', + 'dst': 'pop1', + 'delay': 13, + 'data_rate': 100, + 'used_data_rate': 62, + }, + { + 'src': 'pop1', + 'dst': 'pop2', + 'delay': 13, + 'data_rate': 100, + 'used_data_rate': 62, + }, + { + 'src': 'pop2', + 'dst': 'pop0', + 'delay': 13, + 'data_rate': 100, + 'used_data_rate': 62, + }, + ], + }, + sfcs={ + 'sfc_1': ['a', 'b', 'c'], + }, + service_functions={ + 'a': { + 'processing_delay': 0.5 + }, + 'b': { + 'processing_delay': 0.3 + }, + 'c': { + 'processing_delay': 0.2 + }, + }, + traffic={ + 'pop0': { + 'sfc_1': { + 'a': 1, + 'b': 1, + }, + }, + 'pop1': { + 'sfc_1': { + 'b': 1, + 'c': 1, + }, + }, + 'pop2': { + 'sfc_1': { + 'c': 1, + }, + }, + }, + network_stats={ + 'total_flows': 136, + 'successful_flows': 30, + 'dropped_flows': 3, + 'in_network_flows': 32, + 'avg_end2end_delay': 21, + 'run_avg_end2end_delay': 42, + 'run_total_processed_traffic': dict(), + 'run_avg_path_delay': 22, + 'processed_traffic': dict() + }, + placement={ + 'pop0': ['a'], + 'pop1': ['b'], + 'pop2': ['c'] + } + ) + return state diff --git a/src/siminterface/simulator.py b/src/siminterface/simulator.py index f9674717c..fec8ebad8 100644 --- a/src/siminterface/simulator.py +++ b/src/siminterface/simulator.py @@ -13,7 +13,7 @@ from coordsim.writer.writer import ResultWriter from coordsim.trace_processor.trace_processor import TraceProcessor from coordsim.traffic_predictor.traffic_predictor import TrafficPredictor -from coordsim.traffic_predictor.lstm_predictor import LSTM_Predictor +# from coordsim.traffic_predictor.lstm_predictor import LSTM_Predictor from coordsim.controller import * logger = logging.getLogger(__name__) @@ -63,9 +63,9 @@ def __init__(self, network_file, service_functions_file, config_file, resource_f self.trace = reader.get_trace(trace_path) self.lstm_predictor = None - if 'lstm_prediction' in self.config and self.config['lstm_prediction']: - self.lstm_predictor = LSTM_Predictor(self.trace, params=self.params, - weights_dir=self.config['lstm_weights']) + # if 'lstm_prediction' in self.config and self.config['lstm_prediction']: + # self.lstm_predictor = LSTM_Predictor(self.trace, params=self.params, + # weights_dir=self.config['lstm_weights']) def __del__(self): # write dropped flow locs to yaml diff --git a/src/spinterface/__init__.py b/src/spinterface/__init__.py new file mode 100644 index 000000000..2ef188e1e --- /dev/null +++ b/src/spinterface/__init__.py @@ -0,0 +1,7 @@ +""" Package to specify the coordinator - simulator interface +""" +from .spinterface import SimulatorState +from .spinterface import SimulatorAction +from .spinterface import SimulatorInterface + +__all__ = ['SimulatorAction', 'SimulatorInterface', 'SimulatorState'] diff --git a/src/spinterface/spinterface.py b/src/spinterface/spinterface.py new file mode 100644 index 000000000..c47bd4784 --- /dev/null +++ b/src/spinterface/spinterface.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +""" +Module to abstract interaction between simulator and S&P algorithm +""" + + +class SimulatorAction: + """ + Defines the actions to apply to the simulator environment. + """ + + def __init__(self, + placement: dict, + scheduling: dict): + """initializes all properties since this is a data class + + Parameters + ---------- + placement : dict + { + 'node id' : [list of SF ids] + } + + << Schedule: Must include traffic distribution for all possible nodes. Even those that have a value of zero >> + The Sum of probabilities for each node of each SF needs to sum to 1.0 + Use :func:`~common.common_functionalities.normalize_scheduling_probabilities` to normalize if needed. + scheduling : dict + { + 'node id' : dict + { + 'SFC id' : dict + { + 'SF id' : dict + { + 'node id' : float (Inclusive of zero values) + } + } + } + } + + Examples + -------- + placement = { + 'pop0': ['a', 'c', 'd'], + 'pop1': ['b', 'c', 'd'], + 'pop2': ['a', 'b'], + } + flow_schedule = { + 'pop0': { + 'sfc_1': { + 'a': { + 'pop0': 0.4, + 'pop1': 0.6, + 'pop2': 0 + }, + 'b': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + }, + 'c': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + } + }, + 'sfc_2': { + 'a': { + 'pop0': 0.4, + 'pop1': 0.6, + 'pop2': 0 + }, + 'b': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + }, + 'c': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + } + }, + 'sfc_3': { + 'a': { + 'pop0': 0.4, + 'pop1': 0.6, + 'pop2': 0 + }, + 'b': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + }, + 'c': { + 'pop0': 0.6, + 'pop1': 0.2, + 'pop2': 0.2 + } + }, + }, + 'pop1': { + ... + }, + } + + simulator_action = SimulationAction(placement, flow_schedule) + """ + self.placement = placement + self.scheduling = scheduling + + def __repr__(self): + return "SimulatorAction({})".format(repr({ + 'placement': self.placement, + 'scheduling': self.scheduling + })) + + def __str__(self): + return f"SimulatorAction(Placement: {self.placement}, Schedule: {self.scheduling})" + + +class SimulatorState: + """ + Defines the state of the simulator environment. + Contains all necessary information for an coordination algorithm. + """ + def __init__(self, network, placement, sfcs, service_functions, traffic, network_stats): + """initializes all properties since this is a data class + + Parameters + ---------- + network : dict + { + 'nodes': [{ + 'id': str, + 'resource': [float], + 'used_resources': [float] + }], + 'edges': [{ + 'src': str, + 'dst': str, + 'delay': int (ms), + 'data_rate': int (Mbit/s), + 'used_data_rate': int (Mbit/s), + }], + } + placement : dict + { + 'node id' : [list of SF ids] + } + sfcs : dict + { + 'sfc_id': list + ['ids (str)'] + }, + + service_functions : dict + { + 'sf_id (str)' : dict + { + 'processing_delay_mean': int (ms), + 'processing_delay_stdev': int (ms) + }, + } + + + << traffic: aggregated data rates of flows arriving at node requesting >> + traffic : dict + { + 'node_id (str)' : dict + { + 'sfc_id (str)': dict + { + 'sf_id (str)': data_rate (int) [Mbit/s] + }, + }, + }, + + network_stats : dict + { + 'total_flows' : int, + 'successful_flows' : int, + 'dropped_flows' : int, + 'in_network_flows' : int + 'avg_end_2_end_delay' : int (ms) + } + """ + self.network = network + self.placement = placement + self.sfcs = sfcs + self.service_functions = service_functions + self.traffic = traffic + self.network_stats = network_stats + + def __str__(self): + return f"SimulatorState(Network nodes: {self.network['nodes']}, Traffic: {dict(self.traffic)}, ...)" + + +class SimulatorInterface: + """ + Defines required method on the simulator object. + """ + def __init__(self, test_mode): + self.test_mode = test_mode + + def init(self, seed: int) -> SimulatorState: + """Creates a new simulation environment. + + Parameters + ---------- + seed : int + Seed for reproducible randomness + + Returns + ------- + state: SimulationStateInterface + """ + raise NotImplementedError + + def apply(self, actions: SimulatorAction) -> SimulatorState: + """Applies set of actions. + + Parameters + ---------- + actions: SimulationAction + + Returns + ------- + state: SimulationStateInterface + """ + raise NotImplementedError