Rewrite toolset.sh in Python

.gitignore

@@ -1,4 +1,5 @@
 .env
+.toolset*.yml
 .idea
 .PKGINFO
 .SIGN.RSA.alpine-*

README.md

@@ -316,7 +316,7 @@ Restricts Pieman to only preparing or upgrading the toolset which is located in
 
 Specifies the time zone of the system.
 
-##### TOOLSET_CODENAME="v2-hermes"
+##### TOOLSET_CODENAME="v3-calculon"
 
 Specifies the toolset codename. The parameter allows users and developers to switch between different toolsets. Each codename is connected to its directory in `${TOOLSET_DIR}` which, in turn, contains the target toolset. When a codename is passed via `${TOOLSET_CODENAME}` but there is no such directory in `${TOOLSET_DIR}`, the process of creating of the directory and installing the toolset into it will be initiated.
 

essentials.sh

@@ -27,7 +27,7 @@ MENDER_CLIENT_REVISION="1.7.x"
 
 PIEMAN_MAJOR_VER=0
 
-PIEMAN_MINOR_VER=10
+PIEMAN_MINOR_VER=11
 
 PYTHON_MAJOR_VER=3
 

helpers/apk.sh

@@ -13,21 +13,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Gets the Alpine Package Keeper (APK) version for the specified version of
-# Alpine Linux.
-# Globals:
-#     PIEMAN_UTILS_DIR
-#     PYTHON
-# Arguments:
-#     Version of Alpine Linux
-# Returns:
-#     Alpine Package Keeper version
-get_apk_tools_version() {
-    local alpine_version=$1
-
-    ${PYTHON} "${PIEMAN_UTILS_DIR}"/apk_tools_version.py --alpine-version="${alpine_version}"
-}
-
 # Runs apk.static to build a chroot environment.
 # Globals:
 #     BASE_PACKAGES

helpers/toolset.sh

@@ -13,21 +13,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Removes
-# * the specified files and directories;
-# * the .partial file in the current directory.
+build_toolset() {
+    ${PYTHON} "${PIEMAN_UTILS_DIR}"/build_toolset.py ${PIEMAN_DIR}/.toolset.yml
+}
+
+# Runs the preprocessor against toolset.yml, located in the root directory of
+# Pieman.
 # Globals:
-#     None
+#     PIEMAN_DIR
+#     PIEMAN_UTILS_DIR
+#     PYTHON
 # Arguments:
-#     Target directory
+#     None
 # Returns:
 #     None
-finalise_installation() {
-    for i in "$@"; do
-        rm -rf "${i}"
-    done
-
-    rm -f .partial
+run_preprocessor_against_toolset_yml() {
+    ${PYTHON} "${PIEMAN_UTILS_DIR}"/preprocessor.py ${PIEMAN_DIR}/toolset.yml ${PIEMAN_DIR}/.toolset.yml
 }
 
 # Gets qemu-user-static 3.1 from Ubuntu 19.04 "Disco Dingo".
@@ -63,40 +64,3 @@ get_qemu_emulation_binary() {
     rm    "$(basename "${package}")"
     rm -r usr
 }
-
-# Checks if the specified Toolset component is partially installed, and if so,
-# cleans up its directory and initializes it for the installation, creating the
-# .partial file there.
-# Globals:
-#     None
-# Arguments:
-#     Target directory
-# Returns:
-#     0 if the specified component directory was initialized
-#     1 if there was no need to initialize the specified component directory
-init_installation_if_needed() {
-    local dir=$1
-
-    create_dir "${dir}"
-    if [ -z "$(ls -A "${dir}")" ] || [ -f "${dir}"/.partial ]; then
-        rm -rf "${dir:?}"/*
-
-        touch "${dir}"/.partial
-
-        return 0
-    fi
-
-    return 1
-}
-
-# Figures out the number of CPU cores which are available on the current
-# machine.
-# Globals:
-#     None
-# Arguments:
-#     None
-# Returns:
-#     Number of available cores
-number_of_cores() {
-    grep -c ^processor /proc/cpuinfo
-}

pieman.sh

@@ -123,7 +123,7 @@ def_bool_var SUDO_REQUIRE_PASSWORD true
 
 def_var TIME_ZONE "Etc/UTC"
 
-def_var TOOLSET_CODENAME "v2-hermes"
+def_var TOOLSET_CODENAME "v3-calculon"
 
 def_var TOOLSET_DIR "${PIEMAN_DIR}/toolset"
 
@@ -173,7 +173,7 @@ EXIT_REQUEST="EXIT"
 # shellcheck disable=SC2034
 REDIS_IS_AVAILABLE=true
 
-TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}"
+export TOOLSET_FULL_PATH="${TOOLSET_DIR}/${TOOLSET_CODENAME}"
 
 SOURCE_DIR=devices/${DEVICE}/${OS}
 
@@ -225,6 +225,9 @@ info "checking toolset ${TOOLSET_CODENAME}"
 if [ ! -d "${TOOLSET_FULL_PATH}" ]; then
     info "building toolset ${TOOLSET_CODENAME} since it does not exist"
 fi
+
+run_preprocessor_against_toolset_yml
+
 . toolset.sh
 
 # shellcheck source=./pieman/pieman/build_status_codes

pieman/bin/build_toolset.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2019 Evgeny Golyshev <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+from argparse import ArgumentParser
+
+from yaml.scanner import ScannerError
+
+from pieman import toolset, util
+
+
+def main():
+    """The main entry point. """
+
+    parser = ArgumentParser()
+    parser.add_argument('yml_file', help='path to the Toolset YAML file')
+    args = parser.parse_args()
+
+    try:
+        os.environ['TOOLSET_FULL_PATH']
+    except KeyError:
+        util.fatal('The TOOLSET_FULL_PATH environment variable is undefined.')
+        sys.exit(1)
+
+    try:
+        toolset_tree = toolset.ToolsetProcessor(args.yml_file)
+    except ScannerError as exp:
+        util.fatal('{}'.format(exp))
+        sys.exit(1)
+    except AttributeError as exp:
+        util.fatal('{}'.format(exp))
+        sys.exit(1)
+    except ModuleNotFoundError as exp:
+        util.fatal('{}'.format(exp))
+        sys.exit(1)
+    except toolset.MissingRequiredFields as exp:
+        util.fatal('{}'.format(exp))
+        sys.exit(1)
+
+    for name, module in toolset_tree:
+        for flavour in module['flavours']:
+            flavour_name = next(iter(flavour))
+            mod = module['imported']
+            mod.run(**flavour[flavour_name])
+
+
+if __name__ == '__main__':
+    main()

pieman/bin/preprocessor.py

@@ -0,0 +1,43 @@
+# Copyright (C) 2019 Evgeny Golyshev <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+from argparse import ArgumentParser
+
+from yaml.scanner import ScannerError
+
+from pieman import toolset
+
+
+def main():
+    """The main entry point. """
+
+    parser = ArgumentParser()
+    parser.add_argument('infile', help='path to the file to be processed')
+    parser.add_argument('outfile', help='path to the result file')
+    args = parser.parse_args()
+
+    try:
+        toolset.PreProcessor(args.infile, args.outfile)
+    except ScannerError as exp:
+        sys.stderr.write('{}\n'.format(exp))
+        sys.exit(1)
+    except toolset.UndefinedVariable as exp:
+        sys.stderr.write('{}\n'.format(exp))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

pieman/pieman/__init__.py

@@ -1,3 +1,3 @@
 """The Pieman tools. """
 
-__version__ = '0.10.0'
+__version__ = '0.11.0'