diff --git a/.gitignore b/.gitignore
index 70722166..9b83e367 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
.env
+.toolset*.yml
.idea
.PKGINFO
.SIGN.RSA.alpine-*
diff --git a/README.md b/README.md
index cdadb577..b01876e8 100644
--- a/README.md
+++ b/README.md
@@ -316,7 +316,7 @@ Restricts Pieman to only preparing or upgrading the toolset which is located in
Specifies the time zone of the system.
-##### TOOLSET_CODENAME="v2-hermes"
+##### TOOLSET_CODENAME="v3-calculon"
Specifies the toolset codename. The parameter allows users and developers to switch between different toolsets. Each codename is connected to its directory in `${TOOLSET_DIR}` which, in turn, contains the target toolset. When a codename is passed via `${TOOLSET_CODENAME}` but there is no such directory in `${TOOLSET_DIR}`, the process of creating of the directory and installing the toolset into it will be initiated.
diff --git a/essentials.sh b/essentials.sh
index ba2e7b37..b1c555f6 100644
--- a/essentials.sh
+++ b/essentials.sh
@@ -27,7 +27,7 @@ MENDER_CLIENT_REVISION="1.7.x"
PIEMAN_MAJOR_VER=0
-PIEMAN_MINOR_VER=10
+PIEMAN_MINOR_VER=11
PYTHON_MAJOR_VER=3
diff --git a/helpers/apk.sh b/helpers/apk.sh
index 249752bf..f829de2f 100644
--- a/helpers/apk.sh
+++ b/helpers/apk.sh
@@ -13,21 +13,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-# Gets the Alpine Package Keeper (APK) version for the specified version of
-# Alpine Linux.
-# Globals:
-# PIEMAN_UTILS_DIR
-# PYTHON
-# Arguments:
-# Version of Alpine Linux
-# Returns:
-# Alpine Package Keeper version
-get_apk_tools_version() {
- local alpine_version=$1
-
- ${PYTHON} "${PIEMAN_UTILS_DIR}"/apk_tools_version.py --alpine-version="${alpine_version}"
-}
-
# Runs apk.static to build a chroot environment.
# Globals:
# BASE_PACKAGES
diff --git a/helpers/toolset.sh b/helpers/toolset.sh
index 572be82f..d4a12c85 100644
--- a/helpers/toolset.sh
+++ b/helpers/toolset.sh
@@ -13,21 +13,22 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-# Removes
-# * the specified files and directories;
-# * the .partial file in the current directory.
+build_toolset() {
+ ${PYTHON} "${PIEMAN_UTILS_DIR}"/build_toolset.py ${PIEMAN_DIR}/.toolset.yml
+}
+
+# Runs the preprocessor against toolset.yml, located in the root directory of
+# Pieman.
# Globals:
-# None
+# PIEMAN_DIR
+# PIEMAN_UTILS_DIR
+# PYTHON
# Arguments:
-# Target directory
+# None
# Returns:
# None
-finalise_installation() {
- for i in "$@"; do
- rm -rf "${i}"
- done
-
- rm -f .partial
+run_preprocessor_against_toolset_yml() {
+ ${PYTHON} "${PIEMAN_UTILS_DIR}"/preprocessor.py ${PIEMAN_DIR}/toolset.yml ${PIEMAN_DIR}/.toolset.yml
}
# Gets qemu-user-static 3.1 from Ubuntu 19.04 "Disco Dingo".
@@ -63,40 +64,3 @@ get_qemu_emulation_binary() {
rm "$(basename "${package}")"
rm -r usr
}
-
-# Checks if the specified Toolset component is partially installed, and if so,
-# cleans up its directory and initializes it for the installation, creating the
-# .partial file there.
-# Globals:
-# None
-# Arguments:
-# Target directory
-# Returns:
-# 0 if the specified component directory was initialized
-# 1 if there was no need to initialize the specified component directory
-init_installation_if_needed() {
- local dir=$1
-
- create_dir "${dir}"
- if [ -z "$(ls -A "${dir}")" ] || [ -f "${dir}"/.partial ]; then
- rm -rf "${dir:?}"/*
-
- touch "${dir}"/.partial
-
- return 0
- fi
-
- return 1
-}
-
-# Figures out the number of CPU cores which are available on the current
-# machine.
-# Globals:
-# None
-# Arguments:
-# None
-# Returns:
-# Number of available cores
-number_of_cores() {
- grep -c ^processor /proc/cpuinfo
-}
diff --git a/pieman.sh b/pieman.sh
index 8f06e396..9f3491f8 100755
--- a/pieman.sh
+++ b/pieman.sh
@@ -123,7 +123,7 @@ def_bool_var SUDO_REQUIRE_PASSWORD true
def_var TIME_ZONE "Etc/UTC"
-def_var TOOLSET_CODENAME "v2-hermes"
+def_var TOOLSET_CODENAME "v3-calculon"
def_var TOOLSET_DIR "${PIEMAN_DIR}/toolset"
@@ -173,7 +173,7 @@ EXIT_REQUEST="EXIT"
# shellcheck disable=SC2034
REDIS_IS_AVAILABLE=true
-TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}"
+export TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}"
SOURCE_DIR=devices/${DEVICE}/${OS}
@@ -225,6 +225,9 @@ info "checking toolset ${TOOLSET_CODENAME}"
if [ ! -d "${TOOLSET_FULL_PATH}" ]; then
info "building toolset ${TOOLSET_CODENAME} since it does not exist"
fi
+
+run_preprocessor_against_toolset_yml
+
. toolset.sh
# shellcheck source=./pieman/pieman/build_status_codes
diff --git a/pieman/bin/build_toolset.py b/pieman/bin/build_toolset.py
new file mode 100644
index 00000000..fe66126b
--- /dev/null
+++ b/pieman/bin/build_toolset.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import sys
+from argparse import ArgumentParser
+
+from yaml.scanner import ScannerError
+
+from pieman import toolset, util
+
+
+def main():
+ """The main entry point. """
+
+ parser = ArgumentParser()
+ parser.add_argument('yml_file', help='path to the Toolset YAML file')
+ args = parser.parse_args()
+
+ try:
+ os.environ['TOOLSET_FULL_PATH']
+ except KeyError:
+ util.fatal('The TOOLSET_FULL_PATH environment variable is undefined.')
+ sys.exit(1)
+
+ try:
+ toolset_tree = toolset.ToolsetProcessor(args.yml_file)
+ except ScannerError as exp:
+ util.fatal('{}'.format(exp))
+ sys.exit(1)
+ except AttributeError as exp:
+ util.fatal('{}'.format(exp))
+ sys.exit(1)
+ except ModuleNotFoundError as exp:
+ util.fatal('{}'.format(exp))
+ sys.exit(1)
+ except toolset.MissingRequiredFields as exp:
+ util.fatal('{}'.format(exp))
+ sys.exit(1)
+
+ for name, module in toolset_tree:
+ for flavour in module['flavours']:
+ flavour_name = next(iter(flavour))
+ mod = module['imported']
+ mod.run(**flavour[flavour_name])
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pieman/bin/preprocessor.py b/pieman/bin/preprocessor.py
new file mode 100644
index 00000000..39730599
--- /dev/null
+++ b/pieman/bin/preprocessor.py
@@ -0,0 +1,43 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import sys
+from argparse import ArgumentParser
+
+from yaml.scanner import ScannerError
+
+from pieman import toolset
+
+
+def main():
+ """The main entry point. """
+
+ parser = ArgumentParser()
+ parser.add_argument('infile', help='path to the file to be processed')
+ parser.add_argument('outfile', help='path to the result file')
+ args = parser.parse_args()
+
+ try:
+ toolset.PreProcessor(args.infile, args.outfile)
+ except ScannerError as exp:
+ sys.stderr.write('{}\n'.format(exp))
+ sys.exit(1)
+ except toolset.UndefinedVariable as exp:
+ sys.stderr.write('{}\n'.format(exp))
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pieman/pieman/__init__.py b/pieman/pieman/__init__.py
index 0c6a4083..ae9b8dbc 100644
--- a/pieman/pieman/__init__.py
+++ b/pieman/pieman/__init__.py
@@ -1,3 +1,3 @@
"""The Pieman tools. """
-__version__ = '0.10.0'
+__version__ = '0.11.0'
diff --git a/pieman/pieman/toolset.py b/pieman/pieman/toolset.py
new file mode 100644
index 00000000..26c602a0
--- /dev/null
+++ b/pieman/pieman/toolset.py
@@ -0,0 +1,161 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os
+import re
+import yaml
+from importlib import import_module
+
+
+class RootNameIsNotValid(Exception):
+ pass
+
+
+class MissingRequiredFields(Exception):
+ def __init__(self, module_name, flavour_name, missing_fields):
+ super().__init__(
+ 'The {} flavour of the {} module misses some of the required '
+ 'fields: '
+ '{}'.format(flavour_name, module_name, ', '.join(missing_fields))
+ )
+
+
+class UndefinedVariable(Exception):
+ def __init__(self, var_name):
+ super().__init__("The variable '{}' is undefined".format(var_name))
+
+
+class PreProcessor:
+ def __init__(self, file_name, new_file_name, root_name='toolset'):
+ with open(file_name, 'r') as infile:
+ self._toolset = yaml.load(infile, Loader=yaml.FullLoader)
+
+ self._root_name = root_name
+ self._tree = None
+
+ self._get_tree()
+
+ self._var_re = re.compile(r'\${([\w\d]+)}')
+
+ self._go_through_all_yml(self._tree, self._toolset)
+
+ new_tree = {self._root_name: self._tree}
+ with open(new_file_name, 'w') as outfile:
+ yaml.dump(new_tree, outfile, default_flow_style=False)
+
+ #
+ # Private methods
+ #
+
+ def _get_tree(self):
+ try:
+ self._tree = self._toolset[self._root_name]
+ except KeyError:
+ raise RootNameIsNotValid
+
+ def _go_through_all_yml(self, parent_node, node, table_names=None,
+ parent_node_name=''):
+ table_names = table_names if table_names else {}
+
+ if isinstance(node, dict):
+ for key, val in node.items():
+ table_names['parent_node_name'] = parent_node_name
+ if not isinstance(val, (dict, list, )):
+ table_names[key] = val
+
+ self._go_through_all_yml(node, val, table_names, key)
+ elif isinstance(node, list):
+ for i in node:
+ self._go_through_all_yml(node, i)
+ else:
+ node_value = parent_node[parent_node_name]
+ while True:
+ if not isinstance(node_value, str):
+ break
+
+ match = self._var_re.search(node_value)
+ if match is None:
+ break
+
+ var_name = match[1]
+ try:
+ value = table_names[var_name]
+ except KeyError:
+ try:
+ value = os.environ[var_name]
+ except KeyError:
+ raise UndefinedVariable(var_name)
+
+ node_value = node_value.replace(match[0], value)
+
+ parent_node[parent_node_name] = node_value
+
+
+class ToolsetProcessor:
+ def __init__(self, file_name):
+ with open(file_name, 'r') as infile:
+ self._toolset = yaml.load(infile, Loader=yaml.FullLoader)
+
+ self._modules = {}
+
+ self._get_root()
+
+ self._process_modules()
+
+ self._validate_modules()
+
+ #
+ # Private methods
+ #
+
+ def _get_root(self):
+ try:
+ self._root = self._toolset['toolset']
+ except KeyError:
+ raise RootNameIsNotValid
+
+ def _process_modules(self):
+ """Raises ModuleNotFoundError if one of the specified modules doesn't
+ exist.
+ """
+
+ for module in self._root:
+ module_name = next(iter(module))
+ mod = self._modules[module_name] = {}
+ mod['imported'] = imported = import_module(
+ '.' + module_name, package='pieman.toolset_modules')
+
+ if imported.FLAVOURS_ENABLED:
+ mod['flavours'] = module[module_name]
+ else:
+ mod['flavours'] = [{'default': module[module_name]}]
+
+ def _validate_modules(self):
+ """Raises AttributeError if the REQUIRED_FIELDS attribute is absent in
+ one of the modules.
+ """
+
+ for module_name, module in self._modules.items():
+ for flavour in module['flavours']:
+ flavour_name = next(iter(flavour))
+ got_fields = set(flavour[flavour_name].keys())
+ required_fields = set(module['imported'].REQUIRED_FIELDS)
+ missing_fields = required_fields - got_fields
+ if missing_fields:
+ raise MissingRequiredFields(
+ module_name, flavour_name, missing_fields)
+
+ def __iter__(self):
+ return iter(self._modules.items())
diff --git a/pieman/bin/apk_tools_version.py b/pieman/pieman/toolset_modules/apk.py
similarity index 58%
rename from pieman/bin/apk_tools_version.py
rename to pieman/pieman/toolset_modules/apk.py
index 170835b5..d7821210 100644
--- a/pieman/bin/apk_tools_version.py
+++ b/pieman/pieman/toolset_modules/apk.py
@@ -1,5 +1,4 @@
-#!/usr/bin/python3
-# Copyright (C) 2018 Evgeny Golyshev
+# Copyright (C) 2019 Evgeny Golyshev
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -14,29 +13,22 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-"""Utility intended to fetch the latest version of the apk-tools-static package
-in the specified version of Alpine Linux. The point is that the
-apk-tools-static version is not frozen in the stable release of Alpine Linux
-and may vary.
-"""
-
-import os
+import os.path
import sys
import time
-from argparse import ArgumentParser
from html.parser import HTMLParser
from urllib.error import URLError, HTTPError
from urllib.parse import urljoin
from urllib.request import urlopen
+from pieman import util
-ARCH = 'armhf'
-ALPINE_VERSION = '3.9'
+FLAVOURS_ENABLED = True
MIRROR = 'http://dl-cdn.alpinelinux.org'
-NAP = 1
+REQUIRED_FIELDS = ('arch', 'version', 'dst', )
RETRIES_NUMBER = 5
@@ -68,21 +60,15 @@ def handle_starttag(self, tag, attrs):
break
-def main():
- """The main entry point. """
+def run(**kwargs):
+ arch = kwargs['arch']
+ version = kwargs['version']
+ dst = os.path.join(os.environ['TOOLSET_FULL_PATH'], kwargs['dst'])
- parser = ArgumentParser()
- parser.add_argument('--alpine-version', default=ALPINE_VERSION,
- help='alpine version', metavar='ALPINE_VERSION')
- parser.add_argument('--arch', default=ARCH,
- help='target architecture', metavar='ARCH')
- parser.add_argument('--mirror', default=MIRROR,
- help='mirror', metavar='MIRROR')
- args = parser.parse_args()
+ util.mkdir(os.path.dirname(dst))
- address = urljoin(args.mirror,
- os.path.join('alpine', 'v' + args.alpine_version,
- 'main', args.arch))
+ address = urljoin(MIRROR, os.path.join('alpine', 'v' + version, 'main',
+ arch))
content = b''
for attempt in range(1, RETRIES_NUMBER + 1):
@@ -90,28 +76,27 @@ def main():
content = urlopen(address).read()
break
except HTTPError as exc:
- sys.stderr.write('{}: request failed (error code {})\n'.
- format(sys.argv[0], exc.code))
+ util.fatal('{}: request failed '
+ '(error code {})'.format(sys.argv[0], exc.code))
except URLError as exc:
- sys.stderr.write('{}: {}\n'.format(sys.argv[0], exc.reason))
+ util.fatal('{}: {}'.format(sys.argv[0], exc.reason))
if attempt != RETRIES_NUMBER:
- sys.stderr.write('Retrying in {} seconds...\n'.format(NAP))
- time.sleep(NAP)
+ util.info('{}: retrying in 1 second...'.format(sys.argv[0]))
+ time.sleep(1)
if content == b'' and attempt == RETRIES_NUMBER:
- sys.stderr.write('Could not request {} after {} attempts\n'.
- format(address, RETRIES_NUMBER))
+ util.fatal('{}: could not request {} after {} '
+ 'attempts'.format(sys.argv[0], address, RETRIES_NUMBER))
sys.exit(1)
parser = CustomHTMLParser(content.decode('utf8'))
apk_tools_version = parser.get_apk_tools_version()
if not apk_tools_version:
- sys.stderr.write('Could not get apk tools version\n')
+ util.fatal('{}: could not get apk tools version'.format(sys.argv[0]))
sys.exit(1)
- print(apk_tools_version)
-
-
-if __name__ == '__main__':
- main()
+ download_link = urljoin(address + '/', 'apk-tools-static-{}.apk'.format(
+ apk_tools_version))
+ util.info('Downloading {}'.format(download_link))
+ util.download(download_link, dst)
diff --git a/pieman/pieman/toolset_modules/debootstrap.py b/pieman/pieman/toolset_modules/debootstrap.py
new file mode 100644
index 00000000..78b88a44
--- /dev/null
+++ b/pieman/pieman/toolset_modules/debootstrap.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+FLAVOURS_ENABLED = True
+
+REQUIRED_FIELDS = ('version', 'dst', )
+
+
+def run(*args, **kwargs):
+ pass
diff --git a/pieman/pieman/toolset_modules/download.py b/pieman/pieman/toolset_modules/download.py
new file mode 100644
index 00000000..2ec009c9
--- /dev/null
+++ b/pieman/pieman/toolset_modules/download.py
@@ -0,0 +1,14 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
diff --git a/pieman/pieman/toolset_modules/mender_artifact.py b/pieman/pieman/toolset_modules/mender_artifact.py
new file mode 100644
index 00000000..2dcd79d1
--- /dev/null
+++ b/pieman/pieman/toolset_modules/mender_artifact.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+FLAVOURS_ENABLED = False
+
+REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', )
+
+
+def run(*args, **kwargs):
+ pass
diff --git a/pieman/pieman/toolset_modules/mender_client.py b/pieman/pieman/toolset_modules/mender_client.py
new file mode 100644
index 00000000..2dcd79d1
--- /dev/null
+++ b/pieman/pieman/toolset_modules/mender_client.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+FLAVOURS_ENABLED = False
+
+REQUIRED_FIELDS = ('version', 'git', 'src', 'dst', )
+
+
+def run(*args, **kwargs):
+ pass
diff --git a/pieman/pieman/toolset_modules/qemu_user_static.py b/pieman/pieman/toolset_modules/qemu_user_static.py
new file mode 100644
index 00000000..468197fd
--- /dev/null
+++ b/pieman/pieman/toolset_modules/qemu_user_static.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+FLAVOURS_ENABLED = True
+
+REQUIRED_FIELDS = ('arch', 'codename', 'dst', )
+
+
+def run(*args, **kwargs):
+ pass
diff --git a/pieman/pieman/toolset_modules/uboot.py b/pieman/pieman/toolset_modules/uboot.py
new file mode 100644
index 00000000..3e57c9eb
--- /dev/null
+++ b/pieman/pieman/toolset_modules/uboot.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2019 Evgeny Golyshev
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+FLAVOURS_ENABLED = True
+
+REQUIRED_FIELDS = ('config', 'toolchain', 'src', 'dst', )
+
+
+def run(*args, **kwargs):
+ pass
diff --git a/pieman/pieman/util.py b/pieman/pieman/util.py
index 7832ee13..cef3470b 100644
--- a/pieman/pieman/util.py
+++ b/pieman/pieman/util.py
@@ -16,11 +16,42 @@
"""Miscellaneous utility functions. """
import logging
+import os
+import sys
+from curses import tparm, tigetstr, setupterm
+from urllib.request import urlretrieve
import redis
+
+setupterm()
+
+
LOGGING_FORMATTER = '%(asctime)s %(levelname)-5.5s %(message)s'
+RED = tparm(tigetstr('setaf'), 1).decode('utf8')
+
+YELLOW = tparm(tigetstr('setaf'), 3).decode('utf8')
+
+RESET = tparm(tigetstr('sgr0')).decode('utf8')
+
+
+def _reporthook(chunk_number, buffer_size, total_size):
+ """It must accept three numeric parameters:
+ - a chunk number;
+ - the maximum size chunks are read in;
+ - the total size of the download (-1 if unknown).
+ """
+
+ readsofar = chunk_number * buffer_size
+ if total_size:
+ percent = readsofar * 100 / total_size
+ status = '\r{:>5.1f}% {:>{n}} / {}'.format(
+ percent, readsofar, total_size, n=len(str(total_size)))
+ sys.stderr.write(status)
+ else: # total size is unknown
+ sys.stderr.write('\rread {}'.format(readsofar))
+
def connect_to_redis(host, port):
"""Connects to the specified Redis server. The function raises on of the
@@ -32,6 +63,27 @@ def connect_to_redis(host, port):
return conn
+def download(url, dst):
+ _, msg = urlretrieve(url, dst + '.part', _reporthook)
+
+ sys.stderr.write('\n')
+
+ os.rename(dst + '.part', dst)
+
+ size = os.stat(dst).st_size
+ if int(msg['Content-Length']) == size:
+ filename = os.path.basename(dst)
+ sys.stderr.write('{} was downloaded successfully\n'.format(filename))
+
+
+def fatal(text):
+ sys.stderr.write('{}fatal{}: {}\n'.format(RED, RESET, text))
+
+
+def info(text):
+ sys.stderr.write('{}info{}: {}\n'.format(YELLOW, RESET, text))
+
+
def init_logger(logger, log_level, log_file_prefix='',
logging_formatter=LOGGING_FORMATTER):
"""Initializes the logger. """
@@ -49,3 +101,12 @@ def init_logger(logger, log_level, log_file_prefix='',
file_handler.setFormatter(formatter)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
+
+
+def mkdir(dir_name):
+ """Creates the specified directory, making parent directories
+ as needed.
+ """
+
+ if not os.path.exists(dir_name):
+ os.makedirs(dir_name)
diff --git a/pieman/setup.py b/pieman/setup.py
index 196f9f52..c6715bda 100644
--- a/pieman/setup.py
+++ b/pieman/setup.py
@@ -12,7 +12,7 @@
setup(name='pieman',
- version='0.10.0',
+ version='0.11.0',
description='Pieman package',
long_description=LONG_DESCRIPTION,
url='https://github.com/tolstoyevsky/pieman',
@@ -21,13 +21,14 @@
maintainer_email='eugulixes@gmail.com',
license='https://gnu.org/licenses/gpl-3.0.txt',
scripts=[
- 'bin/apk_tools_version.py',
'bin/bsc.py',
'bin/bscd.py',
+ 'bin/build_toolset.py',
'bin/check_mutually_exclusive_params.py',
'bin/check_redis.py',
'bin/du.py',
'bin/image_attrs.py',
+ 'bin/preprocessor.py',
],
packages=['pieman'],
include_package_data=True,
diff --git a/toolset.sh b/toolset.sh
index 115dad2c..1dc9069c 100755
--- a/toolset.sh
+++ b/toolset.sh
@@ -13,206 +13,17 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-toolchain_dir="gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf"
-cross_compiler="${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/${toolchain_dir}/bin/arm-linux-gnueabihf-"
-
-toolchain_for_mender_dir="gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf"
-cross_compiler_for_mender="${TOOLSET_FULL_PATH}/mender/${toolchain_for_mender_dir}/bin/arm-linux-gnueabihf-"
-uboot_tools="${TOOLSET_FULL_PATH}/mender/uboot-mender/tools"
-mendersoftware_dir="${TOOLSET_FULL_PATH}"/mender/client/src/mender/vendor/github.com/mendersoftware
-
-info "checking Mender dependencies"
-
-if $(are_mender_dependencies_satisfied); then
- info "Mender dependencies are satisfied"
- mender_dependencies_are_satisfied=true
-else
- info "Mender dependencies are not satisfied"
- mender_dependencies_are_satisfied=false
-fi
-
-info "checking Das U-Boot dependencies"
-
-if ! $(are_uboot_dependencies_satisfied) && [[ ! -d "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}" ]]; then
- fatal "Das U-Boot dependencies are not satisfied"
- exit 1
-else
- info "Das U-Boot dependencies are satisfied"
-fi
-
-if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/qemu-user-static"); then
- info "fetching qemu-user-static"
- pushd "${TOOLSET_FULL_PATH}/qemu-user-static"
- get_qemu_emulation_binary
-
- finalise_installation
- popd
-fi
-
-if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/apk"); then
- info "fetching apk.static for Alpine Linux ${ALPINE_VER}"
- pushd "${TOOLSET_FULL_PATH}"/apk
- create_dir "${ALPINE_VER}"
-
- addr=http://dl-cdn.alpinelinux.org/alpine/
- apk_tools_version="$(get_apk_tools_version "${ALPINE_VER}")"
- apk_tools_static="apk-tools-static-${apk_tools_version}.apk"
- apk_tools_static_path="${TOOLSET_FULL_PATH}/apk/${ALPINE_VER}"
-
- wget "${addr}/v${ALPINE_VER}/main/armhf/${apk_tools_static}" -O "${apk_tools_static_path}/${apk_tools_static}"
-
- tar -xzf "${apk_tools_static_path}/${apk_tools_static}" -C "${apk_tools_static_path}"
-
- mv "${apk_tools_static_path}/sbin/apk.static" "${apk_tools_static_path}"
-
- finalise_installation \
- "${apk_tools_static_path}/${apk_tools_static}" \
- "${apk_tools_static_path}/sbin"
- popd
-fi
-
-if [ ! -d "${TOOLSET_FULL_PATH}/debootstrap" ]; then
- info "fetching debootstrap ${DEBOOTSTRAP_VER}"
- pushd "${TOOLSET_FULL_PATH}"
- git clone https://salsa.debian.org/installer-team/debootstrap.git
-
- git -C debootstrap checkout "${DEBOOTSTRAP_VER}"
- popd
-else
- info "checking if the debootstrap version is equal to or higher ${DEBOOTSTRAP_VER}"
-
- if ! is_debootstrap_uptodate; then
- pushd "${TOOLSET_FULL_PATH}"/debootstrap
- info "upgrading debootstrap to ${DEBOOTSTRAP_VER}"
-
- git checkout master
-
- git pull
-
- git checkout ${DEBOOTSTRAP_VER}
- popd
- fi
-fi
-
-if ${mender_dependencies_are_satisfied} && $(init_installation_if_needed "${TOOLSET_FULL_PATH}/mender"); then
- pushd "${TOOLSET_FULL_PATH}/mender"
- info "downloading inventory & identity scripts"
- wget -q -O mender-device-identity "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-device-identity
- wget -q -O mender-inventory-bootloader-integration "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-bootloader-integration
- wget -q -O mender-inventory-hostinfo "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-hostinfo
- wget -q -O mender-inventory-network "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-network
- wget -q -O mender-inventory-os "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-os
- wget -q -O mender-inventory-rootfs-type "${MENDER_CLIENT_REPO}"/"${MENDER_CLIENT_REVISION}"/support/mender-inventory-rootfs-type
-
- info "fetching cross-toolchain for building Das U-Boot (Mender flavour) and Mender client"
- wget "https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/${toolchain_for_mender_dir}.tar.xz" -O "${toolchain_for_mender_dir}.tar.xz"
-
- info "unpacking archive with toolchain for building Das U-Boot (Mender flavour)"
- tar xJf "${toolchain_for_mender_dir}.tar.xz"
- rm "${toolchain_for_mender_dir}.tar.xz"
-
- info "fetching Das U-Boot (Mender flavour) from https://github.com/mendersoftware/uboot-mender.git"
- git clone https://github.com/mendersoftware/uboot-mender.git -b "${UBOOT_MENDER_BRANCH}"
- git -C "uboot-mender" checkout "${UBOOT_MENDER_COMMIT}"
-
- mkdir -p "${mendersoftware_dir}"
-
- info "fetching Mender client from https://github.com/mendersoftware/mender.git"
- git clone https://github.com/mendersoftware/mender.git "${mendersoftware_dir}"/mender
- git -C "${mendersoftware_dir}"/mender checkout "${MENDER_CLIENT_VER}"
-
- info "fetching Mender Artifacts Library from https://github.com/mendersoftware/mender-artifact.git"
- git clone https://github.com/mendersoftware/mender-artifact.git "${mendersoftware_dir}"/mender-artifact
- git -C "${mendersoftware_dir}"/mender-artifact checkout "${MENDER_ARTIFACT_VER}"
- popd
-
- pushd "${TOOLSET_FULL_PATH}/mender/uboot-mender"
- info "building Das U-Boot (Mender flavour)"
-
- ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make --quiet distclean
- ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make rpi_3_32b_defconfig
- ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make PYTHON=python2 -j $(number_of_cores)
- ARCH=arm CROSS_COMPILE="${cross_compiler_for_mender}" make envtools -j $(number_of_cores)
-
- cp "u-boot.bin" "${TOOLSET_FULL_PATH}/mender"
- cp tools/env/fw_printenv "${TOOLSET_FULL_PATH}/mender"
-
- info "generating image for Das U-Boot (Mender flavour)"
- "${uboot_tools}"/mkimage -A arm -T script -C none -n "Boot script" -d "${PIEMAN_DIR}"/files/mender/boot.cmd "${TOOLSET_FULL_PATH}"/mender/boot.scr
- popd
-
- pushd "${mendersoftware_dir}"/mender
- info "building Mender client"
-
- env CGO_ENABLED=1 \
- CC="${cross_compiler_for_mender}"gcc \
- GOARCH=arm \
- GOOS=linux \
- GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build
-
- cp mender "${TOOLSET_FULL_PATH}"/mender
- popd
-
- pushd "${mendersoftware_dir}"/mender-artifact
- info "building Mender Artifacts Library"
-
- env GOPATH="${TOOLSET_FULL_PATH}"/mender/client make build
-
- cp mender-artifact "${TOOLSET_FULL_PATH}"/mender
- popd
-
- pushd "${TOOLSET_FULL_PATH}/mender"
- finalise_installation "${toolchain_for_mender_dir}" client uboot-mender
- popd
-fi
-
-if $(init_installation_if_needed "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"); then
- pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"
- info "fetching cross-toolchain for building Das U-Boot"
- wget "https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/${toolchain_dir}.tar.xz" -O "${toolchain_dir}.tar.xz"
-
- info "unpacking archive with toolchain for building Das U-Boot"
- tar xJf "${toolchain_dir}.tar.xz"
- rm "${toolchain_dir}.tar.xz"
-
- info "fetching Das U-Boot ${UBOOT_VER} from ${UBOOT_URL}"
- git clone --depth=1 -b "v${UBOOT_VER}" https://github.com/u-boot/u-boot.git "u-boot-${UBOOT_VER}"
- popd
-
- pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}/u-boot-${UBOOT_VER}"
- info "building Das U-Boot"
-
- ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_pc_plus_defconfig
-
- # The host system may have both Python 2 and 3 installed. U-Boot
- # depends on Python 2, so it's necessary to specify it explicitly via
- # the PYTHON variable.
- ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores)
-
- cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-pc-plus.bin
-
- ARCH=arm CROSS_COMPILE="${cross_compiler}" make orangepi_zero_defconfig
- ARCH=arm CROSS_COMPILE="${cross_compiler}" PYTHON=python2 make -j $(number_of_cores)
-
- cp u-boot-sunxi-with-spl.bin "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"/u-boot-sunxi-with-spl-for-opi-zero.bin
-
- cp tools/mkimage "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"
- popd
-
- pushd "${TOOLSET_FULL_PATH}/uboot-${UBOOT_VER}"
- finalise_installation "${toolchain_dir}" "u-boot-${UBOOT_VER}" uboot-env
- popd
-fi
+build_toolset
# Correct ownership if needed
-pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")"
-if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then
- info "correcting ownership for ${TOOLSET_FULL_PATH}"
- chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}"
-fi
-
-if ${PREPARE_ONLY_TOOLSET}; then
- success "exiting since PREPARE_ONLY_TOOLSET is set to true"
-
- exit 0
-fi
+#pieman_dir_ownership="$(get_ownership "${PIEMAN_DIR}")"
+#if [ "$(get_ownership "${TOOLSET_FULL_PATH}")" != "${pieman_dir_ownership}" ]; then
+# info "correcting ownership for ${TOOLSET_FULL_PATH}"
+# chown -R "${pieman_dir_ownership}" "${TOOLSET_FULL_PATH}"
+#fi
+#
+#if ${PREPARE_ONLY_TOOLSET}; then
+# success "exiting since PREPARE_ONLY_TOOLSET is set to true"
+#
+# exit 0
+#fi
diff --git a/toolset.yml b/toolset.yml
new file mode 100644
index 00000000..1b3e81a5
--- /dev/null
+++ b/toolset.yml
@@ -0,0 +1,53 @@
+toolset:
+ - apk:
+ - armhf:
+ arch: ${parent_node_name}
+ version: '3.9'
+ dst: apk/${version}/apk-${parent_node_name}.static
+ - debootstrap:
+ - debian:
+ version: 1.0.105
+ git: https://salsa.debian.org/installer-team/debootstrap.git
+ dst: debootstrap/${parent_node_name}
+ - uboot:
+ - mender:
+ version: 988e0ec54
+ config: rpi_3_32b_defconfig
+ git: https://github.com/mendersoftware/uboot-mender.git
+ toolchain: https://releases.linaro.org/components/toolchain/binaries/6.3-2017.05/arm-linux-gnueabihf/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz
+ src: uboot/uboot-mender/u-boot.bin
+ dst: uboot/mender-u-boot.bin
+ - upstream-opi-pc-plus:
+ version: v2019.01
+ config: orangepi_pc_plus_defconfig
+ git: https://github.com/u-boot/u-boot.git
+ depth: 1
+ toolchain: https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz
+ src: uboot/u-boot/u-boot-sunxi-with-spl.bin
+ dst: uboot/u-boot-sunxi-with-spl-for-opi-pc-plus.bin
+ - upstream-opi-zero:
+ config: orangepi_zero_defconfig
+ dir: uboot/u-boot
+ toolchain: uboot/gcc-linaro-6.3.1-2017.05-x86_64_arm-linux-gnueabihf
+ src: uboot/u-boot/u-boot-sunxi-with-spl.bin
+ dst: uboot/u-boot-sunxi-with-spl-for-opi-zero.bin
+ - mender_artifact:
+ version: 2.3.0
+ git: https://github.com/mendersoftware/mender-artifact.git
+ src: mender-artifact/mender-artifact
+ dst: mender/mender-artifact
+ - mender_client:
+ version: 1.7.0
+ git: https://raw.githubusercontent.com/mendersoftware/mender
+ clone_into: ${parent_node_name}
+ src: ${parent_node_name}/mender
+ dst: mender/mender
+ - qemu_user_static:
+ - arm:
+ arch: ${parent_node_name}
+ codename: disco
+ dst: qemu-user-static/qemu-${parent_node_name}-static
+ - aarch64:
+ arch: ${parent_node_name}
+ codename: disco
+ dst: qemu-user-static/qemu-${parent_node_name}-static