diff --git a/.gitignore b/.gitignore index 70722166..9b83e367 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.toolset*.yml .idea .PKGINFO .SIGN.RSA.alpine-* diff --git a/README.md b/README.md index cdadb577..b01876e8 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ Restricts Pieman to only preparing or upgrading the toolset which is located in Specifies the time zone of the system. -##### TOOLSET_CODENAME="v2-hermes" +##### TOOLSET_CODENAME="v3-calculon" Specifies the toolset codename. The parameter allows users and developers to switch between different toolsets. Each codename is connected to its directory in `${TOOLSET_DIR}` which, in turn, contains the target toolset. When a codename is passed via `${TOOLSET_CODENAME}` but there is no such directory in `${TOOLSET_DIR}`, the process of creating of the directory and installing the toolset into it will be initiated. diff --git a/essentials.sh b/essentials.sh index ba2e7b37..b1c555f6 100644 --- a/essentials.sh +++ b/essentials.sh @@ -27,7 +27,7 @@ MENDER_CLIENT_REVISION="1.7.x" PIEMAN_MAJOR_VER=0 -PIEMAN_MINOR_VER=10 +PIEMAN_MINOR_VER=11 PYTHON_MAJOR_VER=3 diff --git a/helpers/apk.sh b/helpers/apk.sh index 249752bf..f829de2f 100644 --- a/helpers/apk.sh +++ b/helpers/apk.sh @@ -13,21 +13,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Gets the Alpine Package Keeper (APK) version for the specified version of -# Alpine Linux. -# Globals: -# PIEMAN_UTILS_DIR -# PYTHON -# Arguments: -# Version of Alpine Linux -# Returns: -# Alpine Package Keeper version -get_apk_tools_version() { - local alpine_version=$1 - - ${PYTHON} "${PIEMAN_UTILS_DIR}"/apk_tools_version.py --alpine-version="${alpine_version}" -} - # Runs apk.static to build a chroot environment. # Globals: # BASE_PACKAGES diff --git a/helpers/toolset.sh b/helpers/toolset.sh index 572be82f..d4a12c85 100644 --- a/helpers/toolset.sh +++ b/helpers/toolset.sh @@ -13,21 +13,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Removes -# * the specified files and directories; -# * the .partial file in the current directory. +build_toolset() { + ${PYTHON} "${PIEMAN_UTILS_DIR}"/build_toolset.py ${PIEMAN_DIR}/.toolset.yml +} + +# Runs the preprocessor against toolset.yml, located in the root directory of +# Pieman. # Globals: -# None +# PIEMAN_DIR +# PIEMAN_UTILS_DIR +# PYTHON # Arguments: -# Target directory +# None # Returns: # None -finalise_installation() { - for i in "$@"; do - rm -rf "${i}" - done - - rm -f .partial +run_preprocessor_against_toolset_yml() { + ${PYTHON} "${PIEMAN_UTILS_DIR}"/preprocessor.py ${PIEMAN_DIR}/toolset.yml ${PIEMAN_DIR}/.toolset.yml } # Gets qemu-user-static 3.1 from Ubuntu 19.04 "Disco Dingo". @@ -63,40 +64,3 @@ get_qemu_emulation_binary() { rm "$(basename "${package}")" rm -r usr } - -# Checks if the specified Toolset component is partially installed, and if so, -# cleans up its directory and initializes it for the installation, creating the -# .partial file there. -# Globals: -# None -# Arguments: -# Target directory -# Returns: -# 0 if the specified component directory was initialized -# 1 if there was no need to initialize the specified component directory -init_installation_if_needed() { - local dir=$1 - - create_dir "${dir}" - if [ -z "$(ls -A "${dir}")" ] || [ -f "${dir}"/.partial ]; then - rm -rf "${dir:?}"/* - - touch "${dir}"/.partial - - return 0 - fi - - return 1 -} - -# Figures out the number of CPU cores which are available on the current -# machine. -# Globals: -# None -# Arguments: -# None -# Returns: -# Number of available cores -number_of_cores() { - grep -c ^processor /proc/cpuinfo -} diff --git a/pieman.sh b/pieman.sh index 8f06e396..9f3491f8 100755 --- a/pieman.sh +++ b/pieman.sh @@ -123,7 +123,7 @@ def_bool_var SUDO_REQUIRE_PASSWORD true def_var TIME_ZONE "Etc/UTC" -def_var TOOLSET_CODENAME "v2-hermes" +def_var TOOLSET_CODENAME "v3-calculon" def_var TOOLSET_DIR "${PIEMAN_DIR}/toolset" @@ -173,7 +173,7 @@ EXIT_REQUEST="EXIT" # shellcheck disable=SC2034 REDIS_IS_AVAILABLE=true -TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}" +export TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}" SOURCE_DIR=devices/${DEVICE}/${OS} @@ -225,6 +225,9 @@ info "checking toolset ${TOOLSET_CODENAME}" if [ ! -d "${TOOLSET_FULL_PATH}" ]; then info "building toolset ${TOOLSET_CODENAME} since it does not exist" fi + +run_preprocessor_against_toolset_yml + . toolset.sh # shellcheck source=./pieman/pieman/build_status_codes diff --git a/pieman/bin/build_toolset.py b/pieman/bin/build_toolset.py new file mode 100644 index 00000000..fe66126b --- /dev/null +++ b/pieman/bin/build_toolset.py @@ -0,0 +1,61 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +from argparse import ArgumentParser + +from yaml.scanner import ScannerError + +from pieman import toolset, util + + +def main(): + """The main entry point. """ + + parser = ArgumentParser() + parser.add_argument('yml_file', help='path to the Toolset YAML file') + args = parser.parse_args() + + try: + os.environ['TOOLSET_FULL_PATH'] + except KeyError: + util.fatal('The TOOLSET_FULL_PATH environment variable is undefined.') + sys.exit(1) + + try: + toolset_tree = toolset.ToolsetProcessor(args.yml_file) + except ScannerError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except AttributeError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except ModuleNotFoundError as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + except toolset.MissingRequiredFields as exp: + util.fatal('{}'.format(exp)) + sys.exit(1) + + for name, module in toolset_tree: + for flavour in module['flavours']: + flavour_name = next(iter(flavour)) + mod = module['imported'] + mod.run(**flavour[flavour_name]) + + +if __name__ == '__main__': + main() diff --git a/pieman/bin/preprocessor.py b/pieman/bin/preprocessor.py new file mode 100644 index 00000000..39730599 --- /dev/null +++ b/pieman/bin/preprocessor.py @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from argparse import ArgumentParser + +from yaml.scanner import ScannerError + +from pieman import toolset + + +def main(): + """The main entry point. """ + + parser = ArgumentParser() + parser.add_argument('infile', help='path to the file to be processed') + parser.add_argument('outfile', help='path to the result file') + args = parser.parse_args() + + try: + toolset.PreProcessor(args.infile, args.outfile) + except ScannerError as exp: + sys.stderr.write('{}\n'.format(exp)) + sys.exit(1) + except toolset.UndefinedVariable as exp: + sys.stderr.write('{}\n'.format(exp)) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/pieman/pieman/__init__.py b/pieman/pieman/__init__.py index 0c6a4083..ae9b8dbc 100644 --- a/pieman/pieman/__init__.py +++ b/pieman/pieman/__init__.py @@ -1,3 +1,3 @@ """The Pieman tools. """ -__version__ = '0.10.0' +__version__ = '0.11.0' diff --git a/pieman/pieman/toolset.py b/pieman/pieman/toolset.py new file mode 100644 index 00000000..26c602a0 --- /dev/null +++ b/pieman/pieman/toolset.py @@ -0,0 +1,161 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import yaml +from importlib import import_module + + +class RootNameIsNotValid(Exception): + pass + + +class MissingRequiredFields(Exception): + def __init__(self, module_name, flavour_name, missing_fields): + super().__init__( + 'The {} flavour of the {} module misses some of the required ' + 'fields: ' + '{}'.format(flavour_name, module_name, ', '.join(missing_fields)) + ) + + +class UndefinedVariable(Exception): + def __init__(self, var_name): + super().__init__("The variable '{}' is undefined".format(var_name)) + + +class PreProcessor: + def __init__(self, file_name, new_file_name, root_name='toolset'): + with open(file_name, 'r') as infile: + self._toolset = yaml.load(infile, Loader=yaml.FullLoader) + + self._root_name = root_name + self._tree = None + + self._get_tree() + + self._var_re = re.compile(r'\${([\w\d]+)}') + + self._go_through_all_yml(self._tree, self._toolset) + + new_tree = {self._root_name: self._tree} + with open(new_file_name, 'w') as outfile: + yaml.dump(new_tree, outfile, default_flow_style=False) + + # + # Private methods + # + + def _get_tree(self): + try: + self._tree = self._toolset[self._root_name] + except KeyError: + raise RootNameIsNotValid + + def _go_through_all_yml(self, parent_node, node, table_names=None, + parent_node_name=''): + table_names = table_names if table_names else {} + + if isinstance(node, dict): + for key, val in node.items(): + table_names['parent_node_name'] = parent_node_name + if not isinstance(val, (dict, list, )): + table_names[key] = val + + self._go_through_all_yml(node, val, table_names, key) + elif isinstance(node, list): + for i in node: + self._go_through_all_yml(node, i) + else: + node_value = parent_node[parent_node_name] + while True: + if not isinstance(node_value, str): + break + + match = self._var_re.search(node_value) + if match is None: + break + + var_name = match[1] + try: + value = table_names[var_name] + except KeyError: + try: + value = os.environ[var_name] + except KeyError: + raise UndefinedVariable(var_name) + + node_value = node_value.replace(match[0], value) + + parent_node[parent_node_name] = node_value + + +class ToolsetProcessor: + def __init__(self, file_name): + with open(file_name, 'r') as infile: + self._toolset = yaml.load(infile, Loader=yaml.FullLoader) + + self._modules = {} + + self._get_root() + + self._process_modules() + + self._validate_modules() + + # + # Private methods + # + + def _get_root(self): + try: + self._root = self._toolset['toolset'] + except KeyError: + raise RootNameIsNotValid + + def _process_modules(self): + """Raises ModuleNotFoundError if one of the specified modules doesn't + exist. + """ + + for module in self._root: + module_name = next(iter(module)) + mod = self._modules[module_name] = {} + mod['imported'] = imported = import_module( + '.' + module_name, package='pieman.toolset_modules') + + if imported.FLAVOURS_ENABLED: + mod['flavours'] = module[module_name] + else: + mod['flavours'] = [{'default': module[module_name]}] + + def _validate_modules(self): + """Raises AttributeError if the REQUIRED_FIELDS attribute is absent in + one of the modules. + """ + + for module_name, module in self._modules.items(): + for flavour in module['flavours']: + flavour_name = next(iter(flavour)) + got_fields = set(flavour[flavour_name].keys()) + required_fields = set(module['imported'].REQUIRED_FIELDS) + missing_fields = required_fields - got_fields + if missing_fields: + raise MissingRequiredFields( + module_name, flavour_name, missing_fields) + + def __iter__(self): + return iter(self._modules.items()) diff --git a/pieman/bin/apk_tools_version.py b/pieman/pieman/toolset_modules/apk.py similarity index 58% rename from pieman/bin/apk_tools_version.py rename to pieman/pieman/toolset_modules/apk.py index 170835b5..d7821210 100644 --- a/pieman/bin/apk_tools_version.py +++ b/pieman/pieman/toolset_modules/apk.py @@ -1,5 +1,4 @@ -#!/usr/bin/python3 -# Copyright (C) 2018 Evgeny Golyshev +# Copyright (C) 2019 Evgeny Golyshev # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,29 +13,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Utility intended to fetch the latest version of the apk-tools-static package -in the specified version of Alpine Linux. The point is that the -apk-tools-static version is not frozen in the stable release of Alpine Linux -and may vary. -""" - -import os +import os.path import sys import time -from argparse import ArgumentParser from html.parser import HTMLParser from urllib.error import URLError, HTTPError from urllib.parse import urljoin from urllib.request import urlopen +from pieman import util -ARCH = 'armhf' -ALPINE_VERSION = '3.9' +FLAVOURS_ENABLED = True MIRROR = 'http://dl-cdn.alpinelinux.org' -NAP = 1 +REQUIRED_FIELDS = ('arch', 'version', 'dst', ) RETRIES_NUMBER = 5 @@ -68,21 +60,15 @@ def handle_starttag(self, tag, attrs): break -def main(): - """The main entry point. """ +def run(**kwargs): + arch = kwargs['arch'] + version = kwargs['version'] + dst = os.path.join(os.environ['TOOLSET_FULL_PATH'], kwargs['dst']) - parser = ArgumentParser() - parser.add_argument('--alpine-version', default=ALPINE_VERSION, - help='alpine version', metavar='ALPINE_VERSION') - parser.add_argument('--arch', default=ARCH, - help='target architecture', metavar='ARCH') - parser.add_argument('--mirror', default=MIRROR, - help='mirror', metavar='MIRROR') - args = parser.parse_args() + util.mkdir(os.path.dirname(dst)) - address = urljoin(args.mirror, - os.path.join('alpine', 'v' + args.alpine_version, - 'main', args.arch)) + address = urljoin(MIRROR, os.path.join('alpine', 'v' + version, 'main', + arch)) content = b'' for attempt in range(1, RETRIES_NUMBER + 1): @@ -90,28 +76,27 @@ def main(): content = urlopen(address).read() break except HTTPError as exc: - sys.stderr.write('{}: request failed (error code {})\n'. - format(sys.argv[0], exc.code)) + util.fatal('{}: request failed ' + '(error code {})'.format(sys.argv[0], exc.code)) except URLError as exc: - sys.stderr.write('{}: {}\n'.format(sys.argv[0], exc.reason)) + util.fatal('{}: {}'.format(sys.argv[0], exc.reason)) if attempt != RETRIES_NUMBER: - sys.stderr.write('Retrying in {} seconds...\n'.format(NAP)) - time.sleep(NAP) + util.info('{}: retrying in 1 second...'.format(sys.argv[0])) + time.sleep(1) if content == b'' and attempt == RETRIES_NUMBER: - sys.stderr.write('Could not request {} after {} attempts\n'. - format(address, RETRIES_NUMBER)) + util.fatal('{}: could not request {} after {} ' + 'attempts'.format(sys.argv[0], address, RETRIES_NUMBER)) sys.exit(1) parser = CustomHTMLParser(content.decode('utf8')) apk_tools_version = parser.get_apk_tools_version() if not apk_tools_version: - sys.stderr.write('Could not get apk tools version\n') + util.fatal('{}: could not get apk tools version'.format(sys.argv[0])) sys.exit(1) - print(apk_tools_version) - - -if __name__ == '__main__': - main() + download_link = urljoin(address + '/', 'apk-tools-static-{}.apk'.format( + apk_tools_version)) + util.info('Downloading {}'.format(download_link)) + util.download(download_link, dst) diff --git a/pieman/pieman/toolset_modules/debootstrap.py b/pieman/pieman/toolset_modules/debootstrap.py new file mode 100644 index 00000000..78b88a44 --- /dev/null +++ b/pieman/pieman/toolset_modules/debootstrap.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('version', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/download.py b/pieman/pieman/toolset_modules/download.py new file mode 100644 index 00000000..2ec009c9 --- /dev/null +++ b/pieman/pieman/toolset_modules/download.py @@ -0,0 +1,14 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/pieman/pieman/toolset_modules/mender_artifact.py b/pieman/pieman/toolset_modules/mender_artifact.py new file mode 100644 index 00000000..2dcd79d1 --- /dev/null +++ b/pieman/pieman/toolset_modules/mender_artifact.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = False + +REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/mender_client.py b/pieman/pieman/toolset_modules/mender_client.py new file mode 100644 index 00000000..2dcd79d1 --- /dev/null +++ b/pieman/pieman/toolset_modules/mender_client.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = False + +REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/qemu_user_static.py b/pieman/pieman/toolset_modules/qemu_user_static.py new file mode 100644 index 00000000..468197fd --- /dev/null +++ b/pieman/pieman/toolset_modules/qemu_user_static.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('arch', 'codename', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/toolset_modules/uboot.py b/pieman/pieman/toolset_modules/uboot.py new file mode 100644 index 00000000..3e57c9eb --- /dev/null +++ b/pieman/pieman/toolset_modules/uboot.py @@ -0,0 +1,22 @@ +# Copyright (C) 2019 Evgeny Golyshev +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +FLAVOURS_ENABLED = True + +REQUIRED_FIELDS = ('config', 'toolchain', 'src', 'dst', ) + + +def run(*args, **kwargs): + pass diff --git a/pieman/pieman/util.py b/pieman/pieman/util.py index 7832ee13..cef3470b 100644 --- a/pieman/pieman/util.py +++ b/pieman/pieman/util.py @@ -16,11 +16,42 @@ """Miscellaneous utility functions. """ import logging +import os +import sys +from curses import tparm, tigetstr, setupterm +from urllib.request import urlretrieve import redis + +setupterm() + + LOGGING_FORMATTER = '%(asctime)s %(levelname)-5.5s %(message)s' +RED = tparm(tigetstr('setaf'), 1).decode('utf8') + +YELLOW = tparm(tigetstr('setaf'), 3).decode('utf8') + +RESET = tparm(tigetstr('sgr0')).decode('utf8') + + +def _reporthook(chunk_number, buffer_size, total_size): + """It must accept three numeric parameters: + - a chunk number; + - the maximum size chunks are read in; + - the total size of the download (-1 if unknown). + """ + + readsofar = chunk_number * buffer_size + if total_size: + percent = readsofar * 100 / total_size + status = '\r{:>5.1f}% {:>{n}} / {}'.format( + percent, readsofar, total_size, n=len(str(total_size))) + sys.stderr.write(status) + else: # total size is unknown + sys.stderr.write('\rread {}'.format(readsofar)) + def connect_to_redis(host, port): """Connects to the specified Redis server. The function raises on of the @@ -32,6 +63,27 @@ def connect_to_redis(host, port): return conn +def download(url, dst): + _, msg = urlretrieve(url, dst + '.part', _reporthook) + + sys.stderr.write('\n') + + os.rename(dst + '.part', dst) + + size = os.stat(dst).st_size + if int(msg['Content-Length']) == size: + filename = os.path.basename(dst) + sys.stderr.write('{} was downloaded successfully\n'.format(filename)) + + +def fatal(text): + sys.stderr.write('{}fatal{}: {}\n'.format(RED, RESET, text)) + + +def info(text): + sys.stderr.write('{}info{}: {}\n'.format(YELLOW, RESET, text)) + + def init_logger(logger, log_level, log_file_prefix='', logging_formatter=LOGGING_FORMATTER): """Initializes the logger. """ @@ -49,3 +101,12 @@ def init_logger(logger, log_level, log_file_prefix='', file_handler.setFormatter(formatter) file_handler.setFormatter(formatter) logger.addHandler(file_handler) + + +def mkdir(dir_name): + """Creates the specified directory, making parent directories + as needed. + """ + + if not os.path.exists(dir_name): + os.makedirs(dir_name) diff --git a/pieman/setup.py b/pieman/setup.py index 196f9f52..c6715bda 100644 --- a/pieman/setup.py +++ b/pieman/setup.py @@ -12,7 +12,7 @@ setup(name='pieman', - version='0.10.0', + version='0.11.0', description='Pieman package', long_description=LONG_DESCRIPTION, url='https://github.com/tolstoyevsky/pieman', @@ -21,13 +21,14 @@ maintainer_email='eugulixes@gmail.com', license='https://gnu.org/licenses/gpl-3.0.txt', scripts=[ - 'bin/apk_tools_version.py', 'bin/bsc.py', 'bin/bscd.py', + 'bin/build_toolset.py', 'bin/check_mutually_exclusive_params.py', 'bin/check_redis.py', 'bin/du.py', 'bin/image_attrs.py', + 'bin/preprocessor.py', ], packages=['pieman'], include_package_data=True, diff --git a/toolset.sh b/toolset.sh index 115dad2c..1dc9069c 100755 --- a/toolset.sh +++ b/toolset.sh @@ -13,206 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -toolchain_dir="gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf" -cross_compiler="${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/${toolchain_dir}/bin/arm-linux-gnueabihf-" - -toolchain_for_mender_dir="gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf" -cross_compiler_for_mender="${TOOLSET_FULL_PATH}/mender/${toolchain_for_mender_dir}/bin/arm-linux-gnueabihf-" -uboot_tools="${TOOLSET_FULL_PATH}/mender/uboot-mender/tools" -mendersoftware_dir="${TOOLSET_FULL_PATH}"/mender/client/src/mender/vendor/github.com/mendersoftware - -info "checking Mender dependencies" - -if $(are_mender_dependencies_satisfied); then - info "Mender dependencies are satisfied" - mender_dependencies_are_satisfied=true -else - info "Mender dependencies are not satisfied" - mender_dependencies_are_satisfied=false -fi - -info "checking Das U-Boot dependencies" - -if ! $(are_uboot_dependencies_satisfied) && [[ ! -d "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" ]]; then - fatal "Das U-Boot dependencies are not satisfied" - exit 1 -else - info "Das U-Boot dependencies are satisfied" -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/qemu-user-static"); then - info "fetching qemu-user-static" - pushd "${TOOLSET_FULL_PATH}/qemu-user-static" - get_qemu_emulation_binary - - finalise_installation - popd -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/apk"); then - info "fetching apk.static for Alpine Linux ${ALPINE_VER}" - pushd "${TOOLSET_FULL_PATH}"/apk - create_dir "${ALPINE_VER}" - - addr=http://dl-cdn.alpinelinux.org/alpine/ - apk_tools_version="$(get_apk_tools_version "${ALPINE_VER}")" - apk_tools_static="apk-tools-static-${apk_tools_version}.apk" - apk_tools_static_path="${TOOLSET_FULL_PATH}/apk/${ALPINE_VER}" - - wget "${addr}/v${ALPINE_VER}/main/armhf/${apk_tools_static}" -O "${apk_tools_static_path}/${apk_tools_static}" - - tar -xzf "${apk_tools_static_path}/${apk_tools_static}" -C "${apk_tools_static_path}" - - mv "${apk_tools_static_path}/sbin/apk.static" "${apk_tools_static_path}" - - finalise_installation \ - "${apk_tools_static_path}/${apk_tools_static}" \ - "${apk_tools_static_path}/sbin" - popd -fi - -if [ ! -d "${TOOLSET_FULL_PATH}/debootstrap" ]; then - info "fetching debootstrap ${DEBOOTSTRAP_VER}" - pushd "${TOOLSET_FULL_PATH}" - git clone https://salsa.debian.org/installer-team/debootstrap.git - - git -C debootstrap checkout "${DEBOOTSTRAP_VER}" - popd -else - info "checking if the debootstrap version is equal to or higher ${DEBOOTSTRAP_VER}" - - if ! is_debootstrap_uptodate; then - pushd "${TOOLSET_FULL_PATH}"/debootstrap - info "upgrading debootstrap to ${DEBOOTSTRAP_VER}" - - git checkout master - - git pull - - git checkout ${DEBOOTSTRAP_VER} - popd - fi -fi - -if ${mender_dependencies_are_satisfied} && $(init_installation_if_needed "${TOOLSET_FULL_PATH}/mender"); then - pushd "${TOOLSET_FULL_PATH}/mender" - info "downloading inventory & identity scripts" - wget -q -O mender-device-identity "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-device-identity - wget -q -O mender-inventory-bootloader-integration "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-bootloader-integration - wget -q -O mender-inventory-hostinfo "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-hostinfo - wget -q -O mender-inventory-network "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-network - wget -q -O mender-inventory-os "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-os - wget -q -O mender-inventory-rootfs-type "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-rootfs-type - - info "fetching cross-toolchain for building Das U-Boot (Mender flavour) and Mender client" - wget "https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/${toolchain_for_mender_dir}.tar.xz" -O "${toolchain_for_mender_dir}.tar.xz" - - info "unpacking archive with toolchain for building Das U-Boot (Mender flavour)" - tar xJf "${toolchain_for_mender_dir}.tar.xz" - rm "${toolchain_for_mender_dir}.tar.xz" - - info "fetching Das U-Boot (Mender flavour) from https://github.com/mendersoftware/uboot-mender.git" - git clone https://github.com/mendersoftware/uboot-mender.git -b "${UBOOT_MENDER_BRANCH}" - git -C "uboot-mender" checkout "${UBOOT_MENDER_COMMIT}" - - mkdir -p "${mendersoftware_dir}" - - info "fetching Mender client from https://github.com/mendersoftware/mender.git" - git clone https://github.com/mendersoftware/mender.git "${mendersoftware_dir}"/mender - git -C "${mendersoftware_dir}"/mender checkout "${MENDER_CLIENT_VER}" - - info "fetching Mender Artifacts Library from https://github.com/mendersoftware/mender-artifact.git" - git clone https://github.com/mendersoftware/mender-artifact.git "${mendersoftware_dir}"/mender-artifact - git -C "${mendersoftware_dir}"/mender-artifact checkout "${MENDER_ARTIFACT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/mender/uboot-mender" - info "building Das U-Boot (Mender flavour)" - - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make --quiet distclean - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make rpi_3_32b_defconfig - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make PYTHON=python2 -j $(number_of_cores) - ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make envtools -j $(number_of_cores) - - cp "u-boot.bin" "${TOOLSET_FULL_PATH}/mender" - cp tools/env/fw_printenv "${TOOLSET_FULL_PATH}/mender" - - info "generating image for Das U-Boot (Mender flavour)" - "${uboot_tools}"/mkimage -A arm -T script -C none -n "Boot script" -d "${PIEMAN_DIR}"/files/mender/boot.cmd "${TOOLSET_FULL_PATH}"/mender/boot.scr - popd - - pushd "${mendersoftware_dir}"/mender - info "building Mender client" - - env CGO_ENABLED=1 \ - CC="${cross_compiler_for_mender}"gcc \ - GOARCH=arm \ - GOOS=linux \ - GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build - - cp mender "${TOOLSET_FULL_PATH}"/mender - popd - - pushd "${mendersoftware_dir}"/mender-artifact - info "building Mender Artifacts Library" - - env GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build - - cp mender-artifact "${TOOLSET_FULL_PATH}"/mender - popd - - pushd "${TOOLSET_FULL_PATH}/mender" - finalise_installation "${toolchain_for_mender_dir}" client uboot-mender - popd -fi - -if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"); then - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - info "fetching cross-toolchain for building Das U-Boot" - wget "https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/${toolchain_dir}.tar.xz" -O "${toolchain_dir}.tar.xz" - - info "unpacking archive with toolchain for building Das U-Boot" - tar xJf "${toolchain_dir}.tar.xz" - rm "${toolchain_dir}.tar.xz" - - info "fetching Das U-Boot ${UBOOT_VER} from ${UBOOT_URL}" - git clone --depth=1 -b "v${UBOOT_VER}" https://github.com/u-boot/u-boot.git "u-boot-${UBOOT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/u-boot-${UBOOT_VER}" - info "building Das U-Boot" - - ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_pc_plus_defconfig - - # The host system may have both Python 2 and 3 installed. U-Boot - # depends on Python 2, so it's necessary to specify it explicitly via - # the PYTHON variable. - ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores) - - cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-pc-plus.bin - - ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_zero_defconfig - ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores) - - cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-zero.bin - - cp tools/mkimage "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - popd - - pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" - finalise_installation "${toolchain_dir}" "u-boot-${UBOOT_VER}" uboot-env - popd -fi +build_toolset # Correct ownership if needed -pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")" -if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then - info "correcting ownership for ${TOOLSET_FULL_PATH}" - chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}" -fi - -if ${PREPARE_ONLY_TOOLSET}; then - success "exiting since PREPARE_ONLY_TOOLSET is set to true" - - exit 0 -fi +#pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")" +#if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then +# info "correcting ownership for ${TOOLSET_FULL_PATH}" +# chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}" +#fi +# +#if ${PREPARE_ONLY_TOOLSET}; then +# success "exiting since PREPARE_ONLY_TOOLSET is set to true" +# +# exit 0 +#fi diff --git a/toolset.yml b/toolset.yml new file mode 100644 index 00000000..1b3e81a5 --- /dev/null +++ b/toolset.yml @@ -0,0 +1,53 @@ +toolset: + - apk: + - armhf: + arch: ${parent_node_name} + version: '3.9' + dst: apk/${version}/apk-${parent_node_name}.static + - debootstrap: + - debian: + version: 1.0.105 + git: https://salsa.debian.org/installer-team/debootstrap.git + dst: debootstrap/${parent_node_name} + - uboot: + - mender: + version: 988e0ec54 + config: rpi_3_32b_defconfig + git: https://github.com/mendersoftware/uboot-mender.git + toolchain: https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz + src: uboot/uboot-mender/u-boot.bin + dst: uboot/mender-u-boot.bin + - upstream-opi-pc-plus: + version: v2019.01 + config: orangepi_pc_plus_defconfig + git: https://github.com/u-boot/u-boot.git + depth: 1 + toolchain: https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz + src: uboot/u-boot/u-boot-sunxi-with-spl.bin + dst: uboot/u-boot-sunxi-with-spl-for-opi-pc-plus.bin + - upstream-opi-zero: + config: orangepi_zero_defconfig + dir: uboot/u-boot + toolchain: uboot/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf + src: uboot/u-boot/u-boot-sunxi-with-spl.bin + dst: uboot/u-boot-sunxi-with-spl-for-opi-zero.bin + - mender_artifact: + version: 2.3.0 + git: https://github.com/mendersoftware/mender-artifact.git + src: mender-artifact/mender-artifact + dst: mender/mender-artifact + - mender_client: + version: 1.7.0 + git: https://raw.githubusercontent.com/mendersoftware/mender + clone_into: ${parent_node_name} + src: ${parent_node_name}/mender + dst: mender/mender + - qemu_user_static: + - arm: + arch: ${parent_node_name} + codename: disco + dst: qemu-user-static/qemu-${parent_node_name}-static + - aarch64: + arch: ${parent_node_name} + codename: disco + dst: qemu-user-static/qemu-${parent_node_name}-static