From adcff5f770b45f00301a4cb9d23f7152e7e6cfc0 Mon Sep 17 00:00:00 2001 From: Adam Ivora Date: Tue, 14 May 2024 22:14:07 +0200 Subject: [PATCH] Add build files --- .github/workflows/build.yml | 12 +- .gitignore | 3 +- dist/.gitignore | 8 ++ dist/aarch64-linux-toolchain.cmake | 3 + dist/build.py | 110 +++++++++++++++++ dist/build_cvode.py | 76 ++++++++++++ dist/build_libxml2.py | 77 ++++++++++++ dist/build_zlib.py | 73 +++++++++++ dist/lint_files.py | 99 +++++++++++++++ dist/merge_binaries.py | 187 +++++++++++++++++++++++++++++ dist/xsdflatten.py | 68 +++++++++++ 11 files changed, 709 insertions(+), 7 deletions(-) create mode 100644 dist/.gitignore create mode 100644 dist/aarch64-linux-toolchain.cmake create mode 100644 dist/build.py create mode 100644 dist/build_cvode.py create mode 100644 dist/build_libxml2.py create mode 100644 dist/build_zlib.py create mode 100644 dist/lint_files.py create mode 100644 dist/merge_binaries.py create mode 100644 dist/xsdflatten.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 745ef2f..ce1f091 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - run: python3 build/lint_files.py + - run: python3 dist/lint_files.py build: strategy: @@ -42,14 +42,14 @@ jobs: run: | sudo apt-get update sudo apt-get install gcc-aarch64-linux-gnu qemu-user - - run: python build/build_cvode.py ${{ matrix.name }} - - run: python build/build_libxml2.py ${{ matrix.name }} - - run: python build/build_zlib.py ${{ matrix.name }} - - run: python build/build.py ${{ matrix.name }} + - run: python dist/build_cvode.py ${{ matrix.name }} + - run: python dist/build_libxml2.py ${{ matrix.name }} + - run: python dist/build_zlib.py ${{ matrix.name }} + - run: python dist/build.py ${{ matrix.name }} - uses: actions/upload-artifact@v3 with: name: ${{ matrix.name }} - path: build/fmus + path: dist/fmus if-no-files-found: error merge-fmus: diff --git a/.gitignore b/.gitignore index 451102b..d128bcd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __pycache__ /venv/ tests/work/ .vscode -build \ No newline at end of file +build +dist \ No newline at end of file diff --git a/dist/.gitignore b/dist/.gitignore new file mode 100644 index 0000000..a6e19ab --- /dev/null +++ b/dist/.gitignore @@ -0,0 +1,8 @@ +*.tar.gz +cvode-*/ +libxml2-*/ +zlib-*/ +fmi1-*/ +fmi2-*/ +fmi3-*/ +fmus/ \ No newline at end of file diff --git a/dist/aarch64-linux-toolchain.cmake b/dist/aarch64-linux-toolchain.cmake new file mode 100644 index 0000000..4c5bb8c --- /dev/null +++ b/dist/aarch64-linux-toolchain.cmake @@ -0,0 +1,3 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc) diff --git a/dist/build.py b/dist/build.py new file mode 100644 index 0000000..fbbc059 --- /dev/null +++ b/dist/build.py @@ -0,0 +1,110 @@ +# build FMUs and fmusim for all FMI versions + +import os +import shutil +import subprocess +from pathlib import Path +import argparse + + +parent_dir = Path(__file__).parent + +parser = argparse.ArgumentParser() +parser.add_argument( + 'platform', + choices={'x86-windows', 'x86_64-windows', 'x86_64-linux', 'aarch64-linux', 'x86_64-darwin', 'aarch64-darwin'}, + help="Platform to build for, e.g. x86_64-windows" +) +parser.add_argument('--cmake-generator') +args, _ = parser.parse_known_args() + + +def build_fmus(fmi_version, fmi_type=None): + + if fmi_type is not None: + build_dir = parent_dir / f'fmi{fmi_version}-{fmi_type}-{args.platform}' + else: + build_dir = parent_dir / f'fmi{fmi_version}-{args.platform}' + + if build_dir.exists(): + shutil.rmtree(build_dir) + + os.makedirs(build_dir) + + cmake_args = [] + + fmi_platform = args.platform + fmi_architecture, fmi_system = fmi_platform.split('-') + + if fmi_system == 'windows': + + cmake_generator = 'Visual Studio 17 2022' if args.cmake_generator is None else args.cmake_generator + + if fmi_architecture == 'x86': + cmake_args += ['-G', cmake_generator, '-A', 'Win32'] + elif fmi_architecture == 'x86_64': + cmake_args += ['-G', cmake_generator, '-A', 'x64'] + + elif fmi_platform == 'aarch64-linux': + + toolchain_file = parent_dir / 'aarch64-linux-toolchain.cmake' + cmake_args += ['-D', f'CMAKE_TOOLCHAIN_FILE={ toolchain_file }'] + + elif fmi_platform == 'x86_64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=x86_64'] + + elif fmi_platform == 'aarch64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=arm64'] + + install_dir = build_dir / 'install' + + if fmi_type is not None: + cmake_args += ['-D', f'FMI_TYPE={fmi_type.upper()}'] + + cmake_args += [ + '-D', f'CMAKE_INSTALL_PREFIX={install_dir}', + '-D', f'FMI_VERSION={fmi_version}', + '-D', f'FMI_ARCHITECTURE={fmi_architecture}', + '-D', 'WITH_FMUSIM=ON', + '-B', build_dir, + parent_dir.parent + ] + + subprocess.check_call(['cmake'] + cmake_args) + subprocess.check_call(['cmake', '--build', build_dir, '--target', 'install', '--config', 'Release']) + + fmus_dir = parent_dir / 'fmus' / f'{fmi_version}.0' + + if fmi_type is not None: + fmus_dir = fmus_dir / fmi_type + + if fmus_dir.exists(): + shutil.rmtree(fmus_dir) + + os.makedirs(fmus_dir) + + fmusim_dir = parent_dir / 'fmus' / f'fmusim-{args.platform}' + + if fmusim_dir.exists(): + shutil.rmtree(fmusim_dir) + + os.makedirs(fmusim_dir) + + for root, dirs, files in os.walk(install_dir): + for file in files: + if file.endswith('.fmu'): + shutil.copyfile(src=install_dir / file, dst=fmus_dir / file) + elif file.startswith('fmusim'): + shutil.copyfile(src=install_dir / file, dst=fmusim_dir / file) + + +if __name__ == '__main__': + + if args.platform in {'x86_64-linux', 'x86-windows', 'x86_64-windows', 'x86_64-darwin'}: + # build_fmus(fmi_version=1, fmi_type='me') + # build_fmus(fmi_version=1, fmi_type='cs') + build_fmus(fmi_version=2) + + build_fmus(fmi_version=3) diff --git a/dist/build_cvode.py b/dist/build_cvode.py new file mode 100644 index 0000000..c324652 --- /dev/null +++ b/dist/build_cvode.py @@ -0,0 +1,76 @@ +from pathlib import Path +from subprocess import check_call +from fmpy.util import download_file +import tarfile +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'platform', + choices={'x86-windows', 'x86_64-windows', 'x86_64-linux', 'aarch64-linux', 'x86_64-darwin', 'aarch64-darwin'}, + help="Platform to build for, e.g. x86_64-windows" +) +args, _ = parser.parse_known_args() + +archive = download_file('https://github.com/LLNL/sundials/releases/download/v6.4.1/cvode-6.4.1.tar.gz', + checksum='0a614e7d7d525d9ec88d5a30c887786d7c3681bd123bb6679fb9a4ec5f4609fe') + +root = Path(__file__).parent + +with tarfile.open(archive) as file: + file.extractall(root) + +build_dir = root / f'cvode-{args.platform}' / 'build' + +install_prefix = root / f'cvode-{args.platform}' / 'install' + +cmake_args = [] + +fmi_platform = args.platform +fmi_architecture, fmi_system = fmi_platform.split('-') + +if fmi_system == 'windows': + + cmake_args = [ + '-G', 'Visual Studio 17 2022', + '-D', 'CMAKE_C_FLAGS_RELEASE=/MT /O2 /Ob2 /DNDEBUG', + '-D', 'CMAKE_C_FLAGS_DEBUG=/MT /Zi /Ob0 /Od /RTC1', + '-A' + ] + + if fmi_architecture == 'x86': + cmake_args.append('Win32') + elif fmi_architecture == 'x86_64': + cmake_args.append('x64') + +elif fmi_platform == 'aarch64-linux': + + toolchain_file = root / 'aarch64-linux-toolchain.cmake' + cmake_args += ['-D', f'CMAKE_TOOLCHAIN_FILE={toolchain_file}'] + +elif fmi_platform == 'x86_64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=x86_64'] + +elif fmi_platform == 'aarch64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=arm64'] + +check_call( + ['cmake'] + + cmake_args + + ['-B', build_dir, + '-D', f'BUILD_SHARED_LIBS=OFF', + '-D', f'BUILD_TESTING=OFF', + '-D', f'EXAMPLES_INSTALL=OFF', + '-D', f'CMAKE_INSTALL_PREFIX={ install_prefix }', + root / 'cvode-6.4.1'] +) + +check_call([ + 'cmake', + '--build', build_dir, + '--config', 'Release', + '--target', 'install' +]) diff --git a/dist/build_libxml2.py b/dist/build_libxml2.py new file mode 100644 index 0000000..4287d0c --- /dev/null +++ b/dist/build_libxml2.py @@ -0,0 +1,77 @@ +from pathlib import Path +from subprocess import check_call +from fmpy.util import download_file +from fmpy import extract +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'platform', + choices={'x86-windows', 'x86_64-windows', 'x86_64-linux', 'aarch64-linux', 'x86_64-darwin', 'aarch64-darwin'}, + help="Platform to build for, e.g. x86_64-windows" +) +(args, _) = parser.parse_known_args() + +archive = download_file('https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.5.zip', + checksum='711675470075cc85ba450f56aff7424f1ecdef00bc5d1d5dced3ffecd1a9b772') + +root = Path(__file__).parent + +extract(archive, root) + +build_dir = root / f'libxml2-{args.platform}' / 'build' + +install_prefix = root / f'libxml2-{args.platform}' / 'install' + +cmake_args = [] + +fmi_platform = args.platform +fmi_architecture, fmi_system = fmi_platform.split('-') + +if fmi_system == 'windows': + + cmake_args = [ + '-G', 'Visual Studio 17 2022', + '-D', 'CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded', + '-A' + ] + + if fmi_architecture == 'x86': + cmake_args.append('Win32') + elif fmi_architecture == 'x86_64': + cmake_args.append('x64') + +elif fmi_platform == 'aarch64-linux': + + toolchain_file = root / 'aarch64-linux-toolchain.cmake' + cmake_args += ['-D', f'CMAKE_TOOLCHAIN_FILE={toolchain_file}'] + +elif fmi_platform == 'x86_64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=x86_64'] + +elif fmi_platform == 'aarch64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=arm64'] + +check_call( + ['cmake'] + + cmake_args + + ['-B', build_dir, + '-D', f'CMAKE_INSTALL_PREFIX={install_prefix}', + '-D', 'BUILD_SHARED_LIBS=OFF', + '-D', 'LIBXML2_WITH_ICONV=OFF', + '-D', 'LIBXML2_WITH_LZMA=OFF', + '-D', 'LIBXML2_WITH_PYTHON=OFF', + '-D', 'LIBXML2_WITH_ZLIB=OFF', + '-D', 'LIBXML2_WITH_TESTS=OFF', + root / 'libxml2-2.11.5'] +) + +check_call([ + 'cmake', + '--build', build_dir, + '--config', 'Release', + '--target', 'install' +]) diff --git a/dist/build_zlib.py b/dist/build_zlib.py new file mode 100644 index 0000000..c26f7ce --- /dev/null +++ b/dist/build_zlib.py @@ -0,0 +1,73 @@ +import tarfile +from pathlib import Path +from subprocess import check_call +from fmpy.util import download_file +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument( + 'platform', + choices={'x86-windows', 'x86_64-windows', 'x86_64-linux', 'aarch64-linux', 'x86_64-darwin', 'aarch64-darwin'}, + help="Platform to build for, e.g. x86_64-windows" +) +args, _ = parser.parse_known_args() + +archive = download_file('https://www.zlib.net/fossils/zlib-1.3.tar.gz', + checksum='ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e') + +root = Path(__file__).parent + +with tarfile.open(archive) as tf: + tf.extractall(root) + +build_dir = root / f'zlib-{args.platform}' / 'build' + +install_prefix = root / f'zlib-{args.platform}' / 'install' + +cmake_args = [] + +fmi_platform = args.platform +fmi_architecture, fmi_system = fmi_platform.split('-') + +if fmi_system == 'windows': + + cmake_args += ['-G', 'Visual Studio 17 2022', '-A'] + + if fmi_architecture == 'x86': + cmake_args.append('Win32') + elif fmi_architecture == 'x86_64': + cmake_args.append('x64') + + cmake_args += [ + '-D', 'CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded' + #'-D', 'CMAKE_C_FLAGS_DEBUG=/MT /Zi /Ob0 /Od /RTC1', + #'-D', 'CMAKE_C_FLAGS_RELEASE=/MT /O2 /Ob2 /DNDEBUG' + ] + +elif fmi_platform == 'aarch64-linux': + + toolchain_file = root / 'aarch64-linux-toolchain.cmake' + cmake_args += ['-D', f'CMAKE_TOOLCHAIN_FILE={ toolchain_file }'] + +elif fmi_platform == 'x86_64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=x86_64'] + +elif fmi_platform == 'aarch64-darwin': + + cmake_args += ['-D', 'CMAKE_OSX_ARCHITECTURES=arm64'] + +check_call( + ['cmake'] + + cmake_args + + ['-B', build_dir, + '-D', f'CMAKE_INSTALL_PREFIX={ install_prefix }', + root / 'zlib-1.3'] +) + +check_call([ + 'cmake', + '--build', build_dir, + '--config', 'Release', + '--target', 'install' +]) diff --git a/dist/lint_files.py b/dist/lint_files.py new file mode 100644 index 0000000..0449342 --- /dev/null +++ b/dist/lint_files.py @@ -0,0 +1,99 @@ +import os +import sys + + +autofix = False + + +def lint_file(filename): + + messages = [] + + with open(filename, 'r', encoding='utf-8') as file: + + for i, line in enumerate(file): + + c = [ord(c) < 128 for c in line] + + if not all(c): + marker = ''.join([' ' if x else '^' for x in c]) + message = "Non-ASCII characters found:\n%s\n%s" % (line[:-1], marker) + messages.append((message, i)) + + if len(line) > 1 and line[-2] in {' '}: + messages.append(("Whitespace at the end of the line", i)) + + if '\t' in line: + messages.append(("Tab character found", i)) + + if '\r' in line: + messages.append(("Carriage return character found", i)) + + return messages + + +def fix_file(filename): + + lines = [] + + with open(filename, 'r', encoding='utf-8') as file: + + for line in file: + line = line.rstrip() + '\n' + line = line.replace('\t', ' ') + lines.append(line) + + if lines[-1] != '': + lines.append('') + + with open(filename, 'w') as file: + file.writelines(lines) + + +total_problems = 0 + +top = os.path.abspath(__file__) +top = os.path.dirname(top) +top = os.path.dirname(top) + +print("Linting files in %s" % top) + +for root, dirs, files in os.walk(top, topdown=True): + + excluded = ['build', 'fmi1_cs', 'fmi1_me', 'fmi2', 'fmi3', 'fmus', 'fmusim', 'ThirdParty', 'venv'] + + # skip build, git, and cache directories + dirs[:] = [d for d in dirs if d not in excluded and not os.path.basename(d).startswith(('.', '_'))] + + for file in files: + + if not file.lower().endswith(('.h', '.c', '.md', '.html', '.csv', '.txt', '.xml')): + continue + + if file.lower().endswith(('.tab.c', '.yy.c')): + continue # generated files + + filename = os.path.join(root, file) + + print(filename) + + messages = lint_file(filename) + + if messages and autofix: + fix_file(filename) + messages = lint_file(filename) + + if messages: + + print("%d problems found in %s:" % (len(messages), filename)) + print() + + for message, line in messages: + print("line %d: %s" % (line + 1, message)) + print() + + total_problems += len(messages) + +print("Total problems found: %d" % total_problems) + +sys.exit(total_problems) diff --git a/dist/merge_binaries.py b/dist/merge_binaries.py new file mode 100644 index 0000000..db9c128 --- /dev/null +++ b/dist/merge_binaries.py @@ -0,0 +1,187 @@ +from datetime import datetime +from glob import glob + +import pytz +import shutil +import subprocess +import zipfile +from pathlib import Path +from subprocess import check_call +from tempfile import mkdtemp +from shutil import rmtree, make_archive +import os +import fmpy +import jinja2 +import markdown2 +from fmpy import read_csv, plot_result, read_model_description + +root = Path(__file__).parent.parent + +dist_merged = root / 'dist-merged' + +os.makedirs(dist_merged, exist_ok=True) + +fmusim = root / f'dist-{fmpy.platform_tuple}' / f'fmusim-{fmpy.platform_tuple}' / 'fmusim' + +parameters = { + 'BouncingBall': [ + '--output-interval', '0.05', + ], + 'Dahlquist': [ + '--output-interval', '0.2', + ], + 'Feedthrough': [ + '--output-interval', '1', + ], + 'StateSpace': [ + '--output-interval', '1', + ], + 'Resource': [ + '--output-interval', '1', + ], + 'Stair': [ + '--output-interval', '1', + ], + 'VanDerPol': [ + '--output-interval', '0.2', + ] +} + + +def set_tool_version(filename, git_executable='git'): + """ Set the Git tag or hash in the generationTool and generationDateAndTime attributes + if the repo is clean """ + + cwd = os.path.dirname(__file__) + + changed_files = subprocess.check_output([git_executable, 'status', '--porcelain', '--untracked=no'], + cwd=cwd).decode('ascii').strip() + + if changed_files: + return + + version = subprocess.check_output([git_executable, 'tag', '--contains'], cwd=cwd).decode('ascii').strip() + + if not version: + version = subprocess.check_output([git_executable, 'rev-parse', '--short', 'HEAD'], cwd=cwd).decode( + 'ascii').strip() + + if not version: + return + + with open(filename, 'r') as f: + lines = f.read() + + isodate = datetime.now(pytz.utc).isoformat() + + lines = lines.replace('"Reference FMUs (development build)"', + f'"Reference FMUs ({version})"\n generationDateAndTime="{isodate}"') + + with open(filename, 'w') as f: + f.write(lines) + + +def merge_fmus(version): + + for dirpath, dirnames, filenames in os.walk(root / 'dist-x86_64-windows' / version): + + for filename in filenames: + + if not filename.endswith('.fmu'): + continue + + model_name, _ = os.path.splitext(filename) + + tempdir = Path(mkdtemp()) + + platforms = ['x86-windows', 'x86_64-windows', 'x86_64-linux', 'x86_64-darwin'] + + if version == '3.0': + platforms += ['aarch64-linux', 'aarch64-darwin'] + + for platform in platforms: + + platform_fmu = root / f'dist-{platform}' / version / filename + + with zipfile.ZipFile(platform_fmu, 'r') as archive: + archive.extractall(path=tempdir) + + if model_name in parameters: + + output_filename = root / f'dist-{fmpy.platform_tuple}' / version / f'{model_name}_ref.csv' + os.makedirs(tempdir / 'documentation', exist_ok=True) + plot_filename = tempdir / 'documentation' / 'result.svg' + + if version == '1.0/cs': + params = ['--interface-type', 'cs'] + else: + params = ['--interface-type', 'me', '--solver', 'cvode'] + + params += parameters[model_name] + + command = [str(fmusim)] + params + ['--output-file', str(output_filename), str(root / f'dist-{fmpy.platform_tuple}' / version / filename)] + + print(' '.join(command)) + + check_call(command) + + result = read_csv(output_filename) + + # create plot + plot_result(result, markers=True, events=True, filename=plot_filename) + + def get_unit(variable): + if variable.unit is not None: + return variable.unit + elif variable.declaredType is not None: + return variable.declaredType.unit + else: + return '' + + # generate index.html + model_description = read_model_description(root / f'dist-{fmpy.platform_tuple}' / version / filename) + loader = jinja2.FileSystemLoader(searchpath=root) + environment = jinja2.Environment(loader=loader, trim_blocks=True) + template = environment.get_template('template.html') + template.globals.update({'get_unit': get_unit}) + md_file = root / model_name / 'readme.md' + html_file = tempdir / 'documentation' / 'index.html' + content = markdown2.markdown_path(md_file, extras=['tables', 'fenced-code-blocks']) + html = template.render( + model_name=model_name, + content=content, + model_description=model_description, + params=' '.join(params) + ) + with open(html_file, 'w') as f: + f.write(html) + for svg_file in glob(f'{str(root / model_name)}/*.svg'): + shutil.copy(svg_file, tempdir / 'documentation') + + # set tool version + set_tool_version(tempdir / 'modelDescription.xml') + + # create archive + merged_fmu = os.path.join(root, 'dist-merged', version, filename) + make_archive(merged_fmu, 'zip', tempdir) + os.rename(merged_fmu + '.zip', merged_fmu) + + # clean up + rmtree(tempdir, ignore_errors=True) + + +for version in ['1.0/cs', '1.0/me', '2.0', '3.0']: + os.makedirs(dist_merged / version) + merge_fmus(version) + +results_dir = root / 'dist-merged' / 'results' + +os.makedirs(results_dir, exist_ok=True) + +# copy fmusim +for system in ['x86-windows', 'x86_64-windows', 'x86_64-linux', 'aarch64-linux', 'x86_64-darwin', 'aarch64-darwin']: + shutil.copytree(src=root / f'dist-{system}' / f'fmusim-{system}', dst=root / 'dist-merged' / f'fmusim-{system}') + +# copy license and readme +for file in ['LICENSE.txt', 'README.md']: + shutil.copyfile(src=root / file, dst=root / 'dist-merged' / file) diff --git a/dist/xsdflatten.py b/dist/xsdflatten.py new file mode 100644 index 0000000..79fd2c3 --- /dev/null +++ b/dist/xsdflatten.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +""" Adapted from https://github.com/esunder/xsdflatten """ + +import sys +import re +import copy +from lxml import etree + + +def get_includes_from_file(filename): + pattern = re.compile('(